diff --git a/alpenhorn/client/cli.py b/alpenhorn/client/cli.py index 36a068744..c52124a79 100644 --- a/alpenhorn/client/cli.py +++ b/alpenhorn/client/cli.py @@ -6,6 +6,7 @@ import peewee as pw from .. import db +from ..common.logger import echo as echo from ..common.util import start_alpenhorn from ..db import ( ArchiveAcq, diff --git a/alpenhorn/client/group.py b/alpenhorn/client/group.py deleted file mode 100644 index 812bf4d96..000000000 --- a/alpenhorn/client/group.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Alpenhorn client interface for operations on `StorageGroup`s.""" - -import click -import peewee as pw - -from ..db import StorageGroup, StorageNode - - -@click.group(context_settings={"help_option_names": ["-h", "--help"]}) -def cli(): - """Commands operating on storage groups. Use to create, modify, and list groups.""" - - -@cli.command() -@click.argument("group_name", metavar="GROUP") -@click.option("--notes", metavar="NOTES") -def create(group_name, notes): - """Create a storage GROUP and add to database.""" - config_connect() - - try: - StorageGroup.get(name=group_name) - print('Group name "%s" already exists! Try a different name!' % group_name) - exit(1) - except pw.DoesNotExist: - StorageGroup.create(name=group_name, notes=notes) - print('Added group "%s" to database.' % group_name) - - -@cli.command() -def list(): - """List known storage groups.""" - config_connect() - - import tabulate - - data = StorageGroup.select(StorageGroup.name, StorageGroup.notes).tuples() - if data: - print(tabulate.tabulate(data, headers=["Name", "Notes"])) - - -@cli.command() -@click.argument("group_name", metavar="GROUP") -@click.argument("new_name", metavar="NEW-NAME") -def rename(group_name, new_name): - """Change the name of a storage GROUP to NEW-NAME.""" - config_connect() - - try: - group = StorageGroup.get(name=group_name) - try: - StorageGroup.get(name=new_name) - print('Group "%s" already exists.' % new_name) - exit(1) - except pw.DoesNotExist: - group.name = new_name - group.save() - print("Updated.") - except pw.DoesNotExist: - print('Group "%s" does not exist!' % group_name) - exit(1) - - -@cli.command() -@click.argument("group_name", metavar="GROUP") -@click.option("--notes", help="Value for the notes field", metavar="NOTES") -def modify(group_name, notes): - """Change the properties of a storage GROUP.""" - config_connect() - - try: - group = StorageGroup.get(name=group_name) - if notes is not None: - if notes == "": - notes = None - group.notes = notes - group.save() - print("Updated.") - else: - print("Nothing to do.") - except pw.DoesNotExist: - print('Group "%s" does not exist!' % group_name) - exit(1) diff --git a/alpenhorn/client/group/__init__.py b/alpenhorn/client/group/__init__.py new file mode 100644 index 000000000..f0b179bf0 --- /dev/null +++ b/alpenhorn/client/group/__init__.py @@ -0,0 +1,24 @@ +"""Alpenhorn client interface for operations on `StorageGroup`s.""" + +import click +import peewee as pw + +from ...db import StorageGroup, StorageNode + +from .create import create +from .list import list_ +from .modify import modify +from .rename import rename +from .show import show + + +@click.group(context_settings={"help_option_names": ["-h", "--help"]}) +def cli(): + """Manage Storage Groups.""" + + +cli.add_command(create, "create") +cli.add_command(list_, "list") +cli.add_command(modify, "modify") +cli.add_command(rename, "rename") +cli.add_command(show, "show") diff --git a/alpenhorn/client/group/create.py b/alpenhorn/client/group/create.py new file mode 100644 index 000000000..4ae5e3403 --- /dev/null +++ b/alpenhorn/client/group/create.py @@ -0,0 +1,39 @@ +"""alpenhorn group create command""" + +import click +import json +import peewee as pw + +from ...db import database_proxy, StorageGroup +from ..options import client_option, set_io_config +from ..cli import echo + + +@click.command() +@click.argument("group_name", metavar="NAME") +@client_option("io_class", default="Default", show_default=True) +@client_option("io_config") +@client_option("io_var") +@client_option("notes") +def create(group_name, io_class, io_config, io_var, notes): + """Create a new storage group. + + The group will be called NAME, which must not already exist. + """ + + io_config = set_io_config(io_config, io_var, dict()) + + with database_proxy.atomic(): + try: + StorageGroup.get(name=group_name) + raise click.ClickException(f'Group "{group_name}" already exists.') + except pw.DoesNotExist: + pass + + StorageGroup.create( + name=group_name, + notes=notes, + io_class=io_class, + io_config=json.dumps(io_config), + ) + echo(f'Created storage group "{group_name}".') diff --git a/alpenhorn/client/group/list.py b/alpenhorn/client/group/list.py new file mode 100644 index 000000000..98fe8d343 --- /dev/null +++ b/alpenhorn/client/group/list.py @@ -0,0 +1,24 @@ +"""alpenhorn group list command""" + +import click +from tabulate import tabulate +from ...db import StorageGroup +from ..cli import echo + + +@click.command() +def list_(): + """List all storage groups.""" + + data = StorageGroup.select( + StorageGroup.name, StorageGroup.io_class, StorageGroup.notes + ).tuples() + if data: + # Add Default I/O class where needed + data = list(data) + for index, row in enumerate(data): + if row[1] is None: + data[index] = (row[0], "Default", row[2]) + echo(tabulate(data, headers=["Name", "I/O Class", "Notes"])) + else: + echo("No storage groups found.") diff --git a/alpenhorn/client/group/modify.py b/alpenhorn/client/group/modify.py new file mode 100644 index 000000000..6a3d269a5 --- /dev/null +++ b/alpenhorn/client/group/modify.py @@ -0,0 +1,56 @@ +"""alpenhorn group modify command""" + +import json +import click +import peewee as pw + +from ...db import StorageGroup, database_proxy +from ..cli import echo +from ..options import client_option, set_io_config + + +@click.command() +@click.argument("group_name", metavar="GROUP") +@client_option("io_class") +@client_option("io_config") +@client_option("io_var") +@client_option("notes") +def modify(group_name, io_class, io_config, io_var, notes): + """Modify storage group metadata. + + Modifies the metadata of the storage group named GROUP, which must already exist. + + NOTE: to change the name of a storage group, use: + + alpenhorn group rename + """ + + if notes == "": + notes = None + if io_class == "": + io_class = None + + with database_proxy.atomic(): + try: + group = StorageGroup.get(name=group_name) + except pw.DoesNotExist: + raise click.ClickException(f'Storage group "{group_name}" does not exist.') + + io_config = set_io_config(io_config, io_var, group.io_config) + + # collect the updated parameters + updates = dict() + if notes != group.notes: + updates["notes"] = notes + if io_class != group.io_class: + updates["io_class"] = io_class + if io_config != group.io_config: + updates["io_config"] = json.dumps(io_config) + + # Update if necessary. + if updates: + update = StorageGroup.update(**updates).where(StorageGroup.id == group.id) + update.execute() + echo("Storage group updated.") + else: + echo("Nothing to do.") diff --git a/alpenhorn/client/group/rename.py b/alpenhorn/client/group/rename.py new file mode 100644 index 000000000..932bea40b --- /dev/null +++ b/alpenhorn/client/group/rename.py @@ -0,0 +1,39 @@ +"""alpenhorn group rename command""" + +import click +import peewee as pw + +from ...db import StorageGroup, database_proxy +from ..cli import echo + + +@click.command() +@click.argument("group_name", metavar="GROUP") +@click.argument("new_name", metavar="NEW_NAME") +def rename(group_name, new_name): + """Rename a storage group. + + The existing storage group named GROUP will be renamed to NEW_NAME. + NEW_NAME must not already be the name of another group. + """ + + if group_name == new_name: + # The easy case + echo("No change.") + else: + with database_proxy.atomic(): + try: + StorageGroup.get(name=new_name) + raise click.ClickException( + f'Storage group "{group_name}" already exists.' + ) + except pw.DoesNotExist: + pass + + try: + group = StorageGroup.get(name=group_name) + group.name = new_name + group.save() + echo(f'Storage group "{group_name}" renamed to "new_name"') + except pw.DoesNotExist: + raise click.ClickException(f"No such storage group: {group_name}.") diff --git a/alpenhorn/client/group/show.py b/alpenhorn/client/group/show.py new file mode 100644 index 000000000..afd9e50a5 --- /dev/null +++ b/alpenhorn/client/group/show.py @@ -0,0 +1,73 @@ +"""alpenhorn group show command""" + +import json +import click +import peewee as pw +from tabulate import tabulate + +from ...db import StorageGroup, StorageNode +from ..cli import echo + + +@click.command() +@click.argument("group_name", metavar="GROUP") +@click.option("--node-details", is_flag=True, help="Show details of listed nodes.") +@click.option("--node-stats", is_flag=True, help="Show usage stats of listed nodes.") +def show(group_name, node_details, node_stats): + """Show details of a storage group. + + Shows details of the storage group named GROUP. + """ + + try: + group = StorageGroup.get(name=group_name) + except pw.DoesNotExist: + raise click.ClickException(f"no such group: {group_name}") + + # Print a report + echo("Storage Group: " + group.name) + echo(" Notes: " + (group.notes if group.notes else "")) + echo(" I/O Class: " + (group.io_class if group.io_class else "Default")) + + echo("\nI/O Config:\n") + if group.io_config: + try: + io_config = json.loads(group.io_config) + if io_config: + # Find length of longest key (but not too long) + keylen = min(max([len(key) for key in io_config]), 30) + for key, value in io_config.items(): + echo(" " + key.rjust(keylen) + ": " + str(value)) + else: + echo(" empty") + except json.JSONDecodeError: + echo("INVALID (JSON decode error)") + else: + echo(" none") + + # List nodes, if any + echo("\nNodes:\n") + nodes = list(StorageNode.select().where(StorageNode.group == group)) + if nodes: + if node_details or node_stats: + if node_details: + data = [ + ( + node.name, + node.host, + "Yes" if node.active else "No", + node.io_class if node.io_class else "Default", + ) + for node in nodes + ] + headers = ["Name", "Host", "Active", "I/O Class"] + if node_stats: + # TODO: add --node-stats support when "alpenhorn node stats" is implemented + raise NotImplementedError() + echo(tabulate(data, headers=headers)) + else: + # simple list + for node in nodes: + echo(" " + node.name) + else: + echo(" none") diff --git a/alpenhorn/client/options.py b/alpenhorn/client/options.py new file mode 100644 index 000000000..da0bf8337 --- /dev/null +++ b/alpenhorn/client/options.py @@ -0,0 +1,148 @@ +"""Common client options and option processing code.""" + +from __future__ import annotations + +import json +import click +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any +del TYPE_CHECKING + + +def client_option(option: str, **extra_kwargs): + """Provide common client options. + + Returns a click.option decorator for the common client option called + `option`. Other keyword arguments are passed on to click.option. + """ + + # Set args for the click.option decorator + if option == "io_class": + args = ( + "io_class", + "-i", + "--class", + ) + kwargs = { + "metavar": "IO_CLASS", + "default": None, + "help": "Set I/O class to IO_CLASS.", + } + elif option == "io_config": + args = ("--io-config",) + kwargs = { + "metavar": "CONFIG", + "default": None, + "help": "Set I/O config to the JSON object literal CONFIG. Any I/O config " + "specified this way may be further modified by --io-var. Setting this to " + "nothing (--io-config=) empties the I/O config.", + } + elif option == "io_var": + args = ("--io-var",) + kwargs = { + "metavar": "VAR=VALUE", + "default": (), + "multiple": True, + "help": "Set I/O config variable VAR to the value VALUE. May be specified " + "multiple times. Modifies any config specified by --io-config. If VALUE " + "is empty (--io-var VAR=), VAR is deleted if present.", + } + elif option == "notes": + args = ("--notes",) + kwargs = {"metavar": "COMMENT", "help": "Set notes to COMMENT."} + else: + raise ValueError(f"Unknown option: {option}") + + # Update kwargs, if given + kwargs.update(extra_kwargs) + + def _decorator(func): + nonlocal args, kwargs + + return click.option(*args, **kwargs)(func) + + return _decorator + + +def set_io_config( + io_config: str | None, io_var: list | tuple, default: str | dict = {} +) -> dict | None: + """Set the I/O config from the command line. + + Processes the --io-config and --io-var options. + + Parameters + ---------- + io_config: + The --io-config parameter data from click + io_var: + The --io-var parameter data from click + default: + If --io-config is None, use this as the default value before applying + io_var edits. If a string, will be JSON decoded. + + Returns + ------- + io_config : dict + The composed I/O config dict, or None if the dict ended up empty. + """ + + # Attempt to compose the I/O config + if io_config == "": + # i.e. the user specified "--io-config=" to delete the I/O config + new_config = {} + elif io_config: + try: + new_config = json.loads(io_config) + except json.JSONDecodeError: + raise click.ClickException("Unable to parse --io-config value as JSON.") + + if not isinstance(new_config, dict): + raise click.ClickException( + "Argument to --io-config must be a JSON object literal." + ) + else: + # Decoding of default is only done if necessary + if isinstance(default, str): + try: + default = json.loads(default) + except json.JSONDecodeError: + raise click.ClickException( + f"Invalid I/O config in database: {default}." + ) + new_config = default + + # Add any --io-var data + for var in io_var: + split_data = var.split("=", 1) + if len(split_data) <= 1: + raise click.ClickException(f'Missing equals sign in "--io-var={var}"') + + if split_data[1] == "": + # "--io-var VAR=" means delete VAR if present + new_config.pop(split_data[0], None) + continue + + # Otherwise, try coercing the value + try: + split_data[1] = int(split_data[1]) + except ValueError: + try: + split_data[1] = float(split_data[1]) + except ValueError: + pass + + # Test jsonifibility. I have not found a way to make click give this function + # something it can't encode, but I don't think this hurts. + try: + json.dumps(dict([split_data])) + except json.JSONEncodeError: + raise click.ClickException(f'Cannot parse argument: "--io-var={var}"') + + # Now set + new_config[split_data[0]] = split_data[1] + + # Return None instead of an empty I/O config + return new_config if new_config else None diff --git a/alpenhorn/common/logger.py b/alpenhorn/common/logger.py index 361d62b9f..013a4b981 100644 --- a/alpenhorn/common/logger.py +++ b/alpenhorn/common/logger.py @@ -42,6 +42,7 @@ default verbosity is 3. May be changed at runtime by calling `set_verbosity`. """ +import click import socket import logging import pathlib @@ -137,7 +138,7 @@ def echo(*args, **kwargs) -> None: Suppresses output when verbosity is less than three. """ - if client_echo: + if _client_echo: return click.echo(*args, **kwargs) diff --git a/tests/client/group/test_create.py b/tests/client/group/test_create.py new file mode 100644 index 000000000..ac149579f --- /dev/null +++ b/tests/client/group/test_create.py @@ -0,0 +1,74 @@ +"""Test CLI: alpenhorn group create""" + +import json +import pytest +from alpenhorn.db import StorageGroup + + +def test_create(clidb, client): + """Test creating a group.""" + + client(0, ["group", "create", "TEST", "--notes=Notes"]) + + # Check created group + group = StorageGroup.get(name="TEST") + assert group.notes == "Notes" + assert group.io_class == "Default" + + +def test_create_existing(clidb, client): + """Test creating a group that already exists.""" + + # Add the group + StorageGroup.create(name="TEST") + + client(1, ["group", "create", "TEST", "--notes=Notes"]) + + # Check that no extra group was created + assert StorageGroup.select().count() == 1 + + +def test_create_ioconifg(clidb, client): + """Test create with --io-config.""" + + client(0, ["group", "create", "TEST", '--io-config={"a": 3, "b": 4}']) + + # Check created io_config + group = StorageGroup.get(name="TEST") + io_config = json.loads(group.io_config) + assert io_config == {"a": 3, "b": 4} + + +def test_create_iovar(clidb, client): + """Test create with --io-var.""" + + result = client( + 0, + ["group", "create", "TEST", "--io-var=a=3", "--io-var", "b=4", "--io-var=a=5"], + ) + + # Check created io_config + group = StorageGroup.get(name="TEST") + io_config = json.loads(group.io_config) + assert io_config == {"a": 5, "b": 4} + + +def test_create_ioconfig_var(clidb, client): + """Test create with --io-config AND --io-var.""" + + result = client( + 0, + [ + "group", + "create", + "TEST", + '--io-config={"a": 6, "b": 7}', + "--io-var=a=8", + "--io-var=c=8.5", + ], + ) + + # Check created io_config + group = StorageGroup.get(name="TEST") + io_config = json.loads(group.io_config) + assert io_config == {"a": 8, "b": 7, "c": 8.5} diff --git a/tests/client/group/test_list.py b/tests/client/group/test_list.py new file mode 100644 index 000000000..4bc4a44e4 --- /dev/null +++ b/tests/client/group/test_list.py @@ -0,0 +1,29 @@ +"""Test CLI: alpenhorn group list""" + +import pytest +from alpenhorn.db import StorageGroup + + +def test_list(clidb, client): + """Test listing groups.""" + + # Make some StorageGroups to list + StorageGroup.create(name="Group1", notes="Note1") + StorageGroup.create(name="Group2", notes="Note2", io_class="Class2") + + result = client(0, ["group", "list"]) + assert "Group1" in result.output + assert "Group2" in result.output + assert "Note1" in result.output + assert "Note2" in result.output + assert "Default" in result.output + assert "Class2" in result.output + + +def test_no_list(clidb, client): + """Test listing no groups.""" + + result = client(0, ["group", "list"]) + + # i.e. the header hasn't been printed + assert "I/O class" not in result.output diff --git a/tests/client/group/test_modify.py b/tests/client/group/test_modify.py new file mode 100644 index 000000000..6c7f76d25 --- /dev/null +++ b/tests/client/group/test_modify.py @@ -0,0 +1,115 @@ +"""Test CLI: alpenhorn group modify""" + +import json +import pytest +from alpenhorn.db import StorageGroup + + +def test_no_modify(clidb, client): + """Test modifying a non-existent group.""" + + client(1, ["group", "modify", "TEST", "--notes=Notes"]) + + # Still nothing. + assert StorageGroup.select().count() == 0 + + +def test_modify_empty(clidb, client): + """Test not modifing a group.""" + + # Add the group + StorageGroup.create(name="TEST", notes="Note", io_class=None) + + # Do nothing successfully. + client(0, ["group", "modify", "TEST"]) + + +def test_modify_no_change(clidb, client): + """Test modify with no change.""" + + StorageGroup.create(name="TEST", notes="Note", io_class=None) + + client(0, ["group", "modify", "TEST", "--notes=Note"]) + + assert StorageGroup.get(name="TEST").notes == "Note" + + +def test_modify(clidb, client): + """Test modify.""" + + StorageGroup.create(name="TEST", notes="Note", io_class=None) + + client(0, ["group", "modify", "TEST", "--notes=New Note", "--class=NewClass"]) + + group = StorageGroup.get(name="TEST") + assert group.notes == "New Note" + assert group.io_class == "NewClass" + + +def test_modify_delete(clidb, client): + """Test deleting metadata with modify.""" + + StorageGroup.create(name="TEST", notes="Note", io_class=None) + + client(0, ["group", "modify", "TEST", "--notes=", "--class="]) + + group = StorageGroup.get(name="TEST") + assert group.notes is None + assert group.io_class is None + + +def test_modify_ioconfig(clidb, client): + """Test updating I/O config with modify.""" + + StorageGroup.create(name="TEST", io_config='{"a": 1, "b": 2, "c": 3}') + + client( + 0, + [ + "group", + "modify", + "TEST", + '--io-config={"a": 4, "b": 5, "c": 6, "d": 7}', + "--io-var", + "b=8", + "--io-var=c=", + ], + ) + + group = StorageGroup.get(name="TEST") + assert json.loads(group.io_config) == {"a": 4, "b": 8, "d": 7} + + +def test_modify_iovar_bad_json(clidb, client): + """--io-var can't be used if the existing I/O config is invalid.""" + + StorageGroup.create(name="TEST", io_config="rawr") + + # Verify the I/O config is invalid + group = StorageGroup.get(name="TEST") + with pytest.raises(json.JSONDecodeError): + json.loads(group.io_config) + + client(1, ["group", "modify", "TEST", "--io-var=a=9"]) + + # I/O config is still broken + group = StorageGroup.get(name="TEST") + with pytest.raises(json.JSONDecodeError): + json.loads(group.io_config) + + +def test_modify_fix_json(clidb, client): + """--io-config can be used to replace invalid JSON in the database.""" + + StorageGroup.create(name="TEST", io_config="rawr") + + # Verify the I/O config is invalid + group = StorageGroup.get(name="TEST") + with pytest.raises(json.JSONDecodeError): + json.loads(group.io_config) + + client(0, ["group", "modify", "TEST", '--io-config={"a": 10}']) + + # Client has fixed the I/O config + group = StorageGroup.get(name="TEST") + assert json.loads(group.io_config) == {"a": 10} diff --git a/tests/client/group/test_rename.py b/tests/client/group/test_rename.py new file mode 100644 index 000000000..2205e4441 --- /dev/null +++ b/tests/client/group/test_rename.py @@ -0,0 +1,47 @@ +"""Test CLI: alpenhorn group rename""" + +import json +import pytest +from alpenhorn.db import StorageGroup + + +def test_no_rename(clidb, client): + """Test rename on a missing group.""" + + client(1, ["group", "rename", "NAME", "NEWNAME"]) + + assert StorageGroup.select().count() == 0 + + +def test_rename(clidb, client): + """Test renaming a group.""" + + # Add the group + StorageGroup.create(name="NAME") + + client(0, ["group", "rename", "NAME", "NEWNAME"]) + + # Check that the rename happened + assert StorageGroup.get(id=1).name == "NEWNAME" + + +def test_idemrename(clidb, client): + """Test renaming a group to it current name.""" + + StorageGroup.create(name="NAME") + + client(0, ["group", "rename", "NAME", "NAME"]) + + +def test_rename_exists(clidb, client): + """Test renaming a group to an exising name.""" + + # Add the groups + StorageGroup.create(name="NAME") + StorageGroup.create(name="NEWNAME") + + client(1, ["group", "rename", "NAME", "NEWNAME"]) + + # Check that the rename didn't happen + assert StorageGroup.get(id=1).name == "NAME" + assert StorageGroup.get(id=2).name == "NEWNAME" diff --git a/tests/client/group/test_show.py b/tests/client/group/test_show.py new file mode 100644 index 000000000..05d28411a --- /dev/null +++ b/tests/client/group/test_show.py @@ -0,0 +1,104 @@ +"""Test CLI: alpenhorn group show""" + +import pytest +from alpenhorn.db import StorageGroup, StorageNode + + +def test_no_show(clidb, client): + """Test showing nothing.""" + + client(1, ["group", "show", "TEST"]) + + +def test_show_defaults(clidb, client): + """Test show with default parameters and no nodes.""" + + # Make a StorageGroup with some nodes in it. + group = StorageGroup.create(name="SGroup") + + result = client(0, ["group", "show", "SGroup"]) + print(result.output) + assert "SGroup" in result.output + assert "Notes" in result.output + assert "Default" in result.output + assert "Nodes" in result.output + + +def test_show_no_io_config(clidb, client): + """Test show with no I/O config.""" + + # Make a StorageGroup with some nodes in it. + group = StorageGroup.create(name="SGroup", notes="Comment", io_class="IOClass") + StorageNode.create(name="Node1", group=group) + StorageNode.create(name="Node2", group=group) + + result = client(0, ["group", "show", "SGroup"]) + print(result.output) + assert "SGroup" in result.output + assert "Comment" in result.output + assert "IOClass" in result.output + assert "I/O Config" in result.output + + +def test_show_empty_io_config(clidb, client): + """Test show with empty I/O config.""" + + # Make a StorageGroup with some nodes in it. + group = StorageGroup.create( + name="SGroup", notes="Comment", io_class="IOClass", io_config="{}" + ) + StorageNode.create(name="Node1", group=group) + StorageNode.create(name="Node2", group=group) + + result = client(0, ["group", "show", "SGroup"]) + print(result.output) + assert "SGroup" in result.output + assert "Comment" in result.output + assert "IOClass" in result.output + assert "I/O Config" in result.output + assert "Node1" in result.output + assert "Node2" in result.output + + +def test_show_io_config(clidb, client): + """Test show with I/O config.""" + + # Make a StorageGroup with some nodes in it. + group = StorageGroup.create( + name="SGroup", + notes="Comment", + io_class="IOClass", + io_config='{"Param1": 1, "Param2": 2}', + ) + + result = client(0, ["group", "show", "SGroup"]) + print(result.output) + assert "SGroup" in result.output + assert "Comment" in result.output + assert "IOClass" in result.output + assert "Param1" in result.output + assert "Param2" in result.output + + +def test_show_node_details(clidb, client): + """Test show --node_details.""" + + # Make a StorageGroup with some nodes in it. + group = StorageGroup.create(name="SGroup", io_class="IOClass") + StorageNode.create(name="Node1", group=group, active=True, host="over_here") + StorageNode.create( + name="Node2", group=group, active=False, host="over_there", io_class="NodeClass" + ) + + result = client(0, ["group", "show", "SGroup", "--node-details"]) + print(result.output) + + assert "Node1" in result.output + assert "Yes" in result.output + assert "over_here" in result.output + assert "Default" in result.output + + assert "Node1" in result.output + assert "No" in result.output + assert "over_there" in result.output + assert "NodeClass" in result.output diff --git a/tests/client/test_client_group.py b/tests/client/test_client_group.py deleted file mode 100644 index 51ffa5c46..000000000 --- a/tests/client/test_client_group.py +++ /dev/null @@ -1,130 +0,0 @@ -""" -test_client_group ----------------------------------- - -Tests for `alpenhorn.client.group` module. -""" - -import re - -import pytest -from click.testing import CliRunner - -# import alpenhorn.client as cli -# import alpenhorn.db as db -# import alpenhorn.storage as st - -# XXX: client is broken -pytest.skip("client is broken", allow_module_level=True) - - -@pytest.fixture -def fixtures(tmpdir): - """Initializes an in-memory Sqlite database with data in tests/fixtures""" - db._connect() - - yield ti.load_fixtures(tmpdir) - - db.database_proxy.close() - - -@pytest.fixture(autouse=True) -def no_cli_init(monkeypatch): - monkeypatch.setattr(cli.group, "config_connect", lambda: None) - - -def test_create_group(fixtures): - """Test the create group command""" - runner = CliRunner() - - help_result = runner.invoke(cli.cli, ["group", "create", "--help"]) - assert help_result.exit_code == 0 - assert "Create a storage GROUP" in help_result.output - - tmpdir = fixtures["root"] - tmpdir.chdir() - result = runner.invoke(cli.cli, args=["group", "create", "group_x"]) - - assert result.exit_code == 0 - assert result.output == 'Added group "group_x" to database.\n' - this_group = st.StorageGroup.get(name="group_x") - assert this_group.name == "group_x" - - # create an already existing node - result = runner.invoke(cli.cli, args=["group", "create", "foo"]) - assert result.exit_code == 1 - assert result.output == 'Group name "foo" already exists! Try a different name!\n' - - -def test_list_groups(fixtures): - """Test the group list command""" - runner = CliRunner() - - help_result = runner.invoke(cli.cli, ["group", "list", "--help"]) - assert help_result.exit_code == 0 - assert "List known storage groups" in help_result.output - - result = runner.invoke(cli.cli, args=["group", "list"]) - assert result.exit_code == 0 - assert re.match( - r"Name +Notes\n" r"-+ +-+\n" r"foo\n" r"bar +Some bar!\n" r"transport\n", - result.output, - re.DOTALL, - ) - - -def test_rename_group(fixtures): - """Test the group rename command""" - runner = CliRunner() - - help_result = runner.invoke(cli.cli, ["group", "rename", "--help"]) - assert help_result.exit_code == 0 - assert "Change the name of a storage GROUP to NEW-NAME" in help_result.output - - result = runner.invoke(cli.cli, args=["group", "rename", "foo", "bar"]) - assert result.exit_code == 1 - assert result.output == 'Group "bar" already exists.\n' - - old_group = st.StorageGroup.get(name="foo") - result = runner.invoke(cli.cli, args=["group", "rename", "foo", "bla"]) - assert result.exit_code == 0 - assert result.output == "Updated.\n" - - new_group = st.StorageGroup.get(name="bla") - assert old_group.id == new_group.id - - -def test_modify_group(fixtures): - """Test the group modify command""" - runner = CliRunner() - - help_result = runner.invoke(cli.cli, ["group", "modify", "--help"]) - assert help_result.exit_code == 0 - assert "Change the properties of a storage GROUP" in help_result.output - - result = runner.invoke(cli.cli, args=["group", "modify", "bla"]) - assert result.exit_code == 1 - assert result.output == 'Group "bla" does not exist!\n' - - result = runner.invoke( - cli.cli, args=["group", "modify", "foo", "--notes=Test test test"] - ) - assert result.exit_code == 0 - assert result.output == "Updated.\n" - - foo_group = st.StorageGroup.get(name="foo") - assert foo_group.notes == "Test test test" - - result = runner.invoke(cli.cli, args=["group", "modify", "foo"]) - assert result.exit_code == 0 - assert result.output == "Nothing to do.\n" - - foo_group = st.StorageGroup.get(name="foo") - assert foo_group.notes == "Test test test" - - result = runner.invoke(cli.cli, args=["group", "modify", "foo", "--notes="]) - assert result.exit_code == 0 - assert result.output == "Updated.\n" - - foo_group = st.StorageGroup.get(name="foo") - assert foo_group.notes is None diff --git a/tests/client/test_options.py b/tests/client/test_options.py new file mode 100644 index 000000000..b0c1e498a --- /dev/null +++ b/tests/client/test_options.py @@ -0,0 +1,97 @@ +"""Test alpenhorn.client.options""" + +import click +import pytest +from alpenhorn.client.options import set_io_config + + +def test_sic_empty(): + """set_io_config with no user input should return default""" + io_config = set_io_config(None, (), {"a": 1, "b": 2}) + assert io_config == {"a": 1, "b": 2} + + +def test_sic_io_config(): + """set_io_config decodes io_config and replace the default""" + + io_config = set_io_config('{"a": 3, "b": 4}', (), {"a": 1, "b": 2}) + assert io_config == {"a": 3, "b": 4} + + +def test_sic_io_var(): + """io_var overrides io_config""" + io_config = set_io_config( + '{"a": 3, "b": 4}', ("a=5", "b=6", "a=7", "c=8"), {"a": 1, "b": 2} + ) + assert io_config == {"a": 7, "b": 6, "c": 8} + + +def test_sic_decodeerror(): + """Check set_io_config JSON decoding error.""" + + with pytest.raises(click.ClickException): + set_io_config("a=9", (), {}) + + +def test_sic_bad_ioconfig(): + """Check set_io_config with wrong JSON type.""" + + with pytest.raises(click.ClickException): + set_io_config("[10, 11]", (), {}) + + +def test_sic_iovar_no_equals(clidb, client): + """Test io_var with no equals sign.""" + + with pytest.raises(click.ClickException): + set_io_config(None, ("a",), {}) + + +def test_create_iovar_equals_equals(clidb, client): + """Test io_var with many equals signs.""" + + io_config = set_io_config(None, ("a=12=13=14",), {}) + assert io_config == {"a": "12=13=14"} + + +def test_sic_coercion(): + """Test coersion of numeric types in set_io_config""" + + io_config = set_io_config('{"a": 15, "b": 16.17}', ("c=18", "d=19.20"), {}) + assert io_config == {"a": 15, "b": 16.17, "c": 18, "d": 19.2} + + +def test_sic_empty_ioconfig(): + """An empty string as io_config is treated as an empty dict.""" + + io_config = set_io_config("", (), {"a": 21, "b": 22}) + assert io_config is None + + +def test_sic_iovar_del(): + """--io-var VAR= should delete VAR if present in the I/O config.""" + + io_config = set_io_config('{"a": 23, "b": 24}', ("a=", "c="), {}) + assert io_config == {"b": 24} + + +def test_sic_str_default(): + """Test a string default to set_io_config""" + + io_config = set_io_config(None, ("a=25",), '{"a": 26, "b": 27}') + assert io_config == {"a": 25, "b": 27} + + +def test_sic_default_decode(): + """Test a decode error in default to set_io_config""" + + with pytest.raises(click.ClickException): + set_io_config(None, ("a=28",), "rawr") + + +def test_sic_default_decode_override(): + """Providing io_config avoids the decode error in set_io_config""" + + io_config = set_io_config("", ("a=29",), "rawr") + + assert io_config == {"a": 29} diff --git a/tests/conftest.py b/tests/conftest.py index 39cd2503e..57ca22507 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,14 @@ """Common fixtures""" import os +import sys +import yaml import pytest import shutil import logging +import pathlib from unittest.mock import patch, MagicMock +from urllib.parse import quote as urlquote import alpenhorn.common.logger from alpenhorn.common import config, extensions @@ -65,6 +69,11 @@ def pytest_configure(config): "used to set the alpenhorn.config for testing. config_dict" "is merged with the default config.", ) + config.addinivalue_line( + "markers", + "clirunner_args(**kwargs): " + "set arguments used to instantiate the click.testing.CliRunner.", + ) @pytest.fixture @@ -577,6 +586,128 @@ def mockgroupandnode(hostname, queue, storagenode, storagegroup, mockio): ), node +@pytest.fixture +def client(request, xfs, cli_config): + """Set up client tests using click + + Yields a wrapper around click.testing.CliRunner().invoke. + The first parameter passed to the wrapper should be + the expected exit code. Other parameters are passed + to CliRunner.invoke (including the list of command + line parameters). + + The wrapper performs rudimentary checks on the result, + then returns the click.result so the caller can inspect + the result further, if desired. + """ + + from click.testing import CliRunner + + marker = request.node.get_closest_marker("clirunner_args") + if marker is not None: + kwargs = marker.kwargs + else: + kwargs = {} + + runner = CliRunner(**kwargs) + + def _cli_wrapper(expected_result, *args, **kwargs): + nonlocal runner + + import traceback + from alpenhorn.client import cli + + result = runner.invoke(cli, *args, **kwargs) + + # Show traceback if one was created + if ( + result.exit_code + and result.exc_info + and type(result.exception) is not SystemExit + ): + traceback.print_exception(*result.exc_info) + + # If unexpected result, show output + if result.exit_code != expected_result: + print(result.output) + + assert result.exit_code == expected_result + if expected_result: + assert type(result.exception) is SystemExit + else: + assert result.exception is None + + return result + + yield _cli_wrapper + + +@pytest.fixture +def clidb_uri(): + """Returns database URI for a shared in-memory database.""" + return "file:clidb?mode=memory&cache=shared" + + +@pytest.fixture +def clidb(clidb_noinit): + """Initiliase a peewee connector to the CLI DB and create tables. + + Yields the connector.""" + + clidb_noinit.create_tables( + [ + StorageGroup, + StorageNode, + StorageTransferAction, + ArchiveAcq, + ArchiveFile, + ArchiveFileCopy, + ArchiveFileCopyRequest, + ] + ) + + yield clidb_noinit + + +@pytest.fixture +def clidb_noinit(clidb_uri): + """Initialise a peewee connector to the empty CLI DB. + + Yields the connector.""" + + import peewee as pw + from alpenhorn.db import database_proxy, EnumField + + # Open + db = pw.SqliteDatabase(clidb_uri, uri=True) + assert db is not None + database_proxy.initialize(db) + EnumField.native = False + + yield db + + db.close() + + +@pytest.fixture +def cli_config(xfs, clidb_uri): + """Fixture creating the config file for CLI tests.""" + + # The config. + # + # The weird value for "url" here gets around playhouse.db_url not + # url-decoding the netloc of the supplied URL. The netloc is used + # as the "database" value, so to get the URI in there, we need to pass + # it as a parameter, which WILL get urldecoded and supercede the empty + # netloc. + config = { + "database": {"url": "sqlite:///?database=" + urlquote(clidb_uri) + "&uri=true"}, + } + + # Put it in a file + xfs.create_file("/etc/alpenhorn/alpenhorn.conf", contents=yaml.dump(config)) + + # Data table fixtures. Each of these will add a row with the specified # data to the appropriate table in the DB, creating the table first if # necessary diff --git a/tests/server/test_service.py b/tests/server/test_service.py index e040c073c..567ea1bfa 100644 --- a/tests/server/test_service.py +++ b/tests/server/test_service.py @@ -36,19 +36,12 @@ sys.path.append(str(pathlib.Path(__file__).parent.joinpath("..", "..", "examples"))) import pattern_importer -# database URI for a shared in-memory database -DB_URI = "file:e2edb?mode=memory&cache=shared" - @pytest.fixture -def e2e_db(xfs, hostname): +def e2e_db(xfs, clidb_noinit, hostname): """Create and populate the DB for the end-to-end test.""" - # Open - db = pw.SqliteDatabase(DB_URI, uri=True) - assert db is not None - database_proxy.initialize(db) - EnumField.native = False + db = clidb_noinit # Create tables db.create_tables( @@ -288,7 +281,7 @@ def _mocked_rsync(from_path, to_dir, size_b, local): @pytest.fixture -def e2e_config(xfs, hostname): +def e2e_config(xfs, hostname, clidb_uri): """Fixture creating the config file for the end-to-end test.""" # The config. @@ -303,7 +296,7 @@ def e2e_config(xfs, hostname): "extensions": [ "pattern_importer", ], - "database": {"url": "sqlite:///?database=" + urlquote(DB_URI) + "&uri=true"}, + "database": {"url": "sqlite:///?database=" + urlquote(clidb_uri) + "&uri=true"}, "logging": {"level": "debug"}, "service": {"num_workers": 0}, }