Make devcontainer networking more friendly; MVP event manager

.devcontainer/devcontainer.compose.yaml

@@ -1,3 +1,4 @@
+name: madsci_dev
 services:
   dev:
     build:
@@ -14,6 +15,8 @@ services:
     entrypoint: /usr/local/share/docker-init.sh
     command: sleep infinity
 
+    network_mode: host
+
     # Uncomment the next four lines if you will use a ptrace-based debuggers like C++, Go, and Rust.
     # cap_add:
     #  - SYS_PTRACE

.devcontainer/devcontainer.json

@@ -44,7 +44,8 @@
 		"ghcr.io/devcontainers-extra/features/pdm:2": {},
 		"ghcr.io/devcontainers-extra/features/vue-cli:2": {},
 		"ghcr.io/devcontainers-extra/features/pre-commit:2": {},
-		"ghcr.io/devcontainers-extra/features/ruff:1": {}
+		"ghcr.io/devcontainers-extra/features/ruff:1": {},
+		"ghcr.io/devcontainers-extra/features/mongosh-homebrew:1": {}
 	},
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [8000],

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
       - id: nbstripout
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.8.1
+    rev: v0.8.3
     hooks:
       # Run the linter.
       - id: ruff

compose.yaml

@@ -0,0 +1,7 @@
+name: madsci_test
+services:
+  mongodb:
+    container_name: mongodb
+    image: mongodb/mongodb-community-server:latest
+    ports:
+      - 27017:27017

madsci/madsci_client/madsci/client/event_client.py

@@ -1,16 +1,20 @@
 """MADSci Event Handling."""
 
 import contextlib
+import inspect
 import logging
+from collections import OrderedDict
 from pathlib import Path
 from typing import Any, Optional, Union
 
+import requests
 from pydantic import ValidationError
 from rich import print
 
 from madsci.common.types.auth_types import OwnershipInfo
 from madsci.common.types.base_types import PathLike
 from madsci.common.types.event_types import Event, EventType
+from madsci.common.utils import threaded_task
 
 
 class EventClient:
@@ -25,6 +29,17 @@ def __init__(
         log_dir: Optional[PathLike] = None,
     ) -> None:
         """Initialize the event logger."""
+        if name:
+            self.name = name
+        else:
+            # * See if there's a calling class we can name after
+            stack = inspect.stack()
+            parent = stack[1][0]
+            if calling_self := parent.f_locals.get("self"):
+                self.name = calling_self.__class__.__name__
+            else:
+                # * No luck, name after EventClient
+                self.name = __name__
         self.name = name if name else __name__
         self.logger = logging.getLogger(self.name)
         self.log_dir = Path(log_dir) if log_dir else Path.home() / ".madsci" / "logs"
@@ -37,41 +52,56 @@ def __init__(
         self.event_server = event_server
         self.source = source
 
-    def get_log(self) -> list[Event]:
+    def get_log(self) -> dict[str, Event]:
         """Read the log"""
-        events = []
+        events = {}
         with self.logfile.open() as log:
             for line in log.readlines():
                 try:
-                    events.append(Event.model_validate_json(line))
+                    event = Event.model_validate_json(line)
+                    events[event.event_id] = event
                 except ValidationError:
                     events.append(Event(event_type=EventType.UNKNOWN, event_data=line))
         return events
 
+    def get_events(self, number: int = 100) -> dict[str, Event]:
+        """Query the event server for a certain number of recent events. If no event server is configured, query the log file instead."""
+        events = OrderedDict()
+        if self.event_server:
+            response = requests.get(self.event_server + "/events", timeout=10)
+            if not response.ok:
+                response.raise_for_status()
+            print(response.json())
+            for key, value in response.json().items():
+                events[key] = Event.model_validate(value)
+            return dict(events)
+        events = self.get_log()
+        selected_events = {}
+        for event in reversed(list(events.values())):
+            selected_events[event.event_id] = event
+            if len(selected_events) >= number:
+                break
+        return selected_events
+
     def log(self, event: Union[Event, Any], level: Optional[int] = None) -> None:
         """Log an event."""
 
-        # * If we've got a string, check if it's a serialized event
+        # * If we've got a string or dict, check if it's a serialized event
         if isinstance(event, str):
             with contextlib.suppress(ValidationError):
                 event = Event.model_validate_json(event)
         if isinstance(event, dict):
             with contextlib.suppress(ValidationError):
                 event = Event.model_validate(**event)
         if not isinstance(event, Event):
-            event = Event(
-                event_type=EventType.LOG,
-                event_data=event,
-            )
+            event = self._new_event_for_log(event, level)
         event.log_level = level if level else event.log_level
         event.source = event.source if event.source is not None else self.source
         self.logger.log(event.log_level, event.model_dump_json())
         if self.logger.getEffectiveLevel() <= event.log_level:
-            print(
-                f"{event.event_timestamp} ({event.log_level}/{event.event_type}): {event.event_data}"
-            )
+            print(f"{event.event_timestamp} ({event.event_type}): {event.event_data}")
         if self.event_server:
-            self.send_event(event)
+            self._send_event(event)
 
     def log_debug(self, event: Union[Event, str]) -> None:
         """Log an event at the debug level."""
@@ -93,9 +123,31 @@ def log_critical(self, event: Union[Event, str]) -> None:
         """Log an event at the critical level."""
         self.log(event, logging.CRITICAL)
 
-    def send_event(self, event: Event) -> None:
+    @threaded_task
+    def _send_event_to_event_server(self, event: Event) -> None:
         """Send an event to the event manager."""
-        raise NotImplementedError()
-
-
-default_event_logger = EventClient()
+        response = requests.post(
+            url=self.event_server + "/event",
+            json=event.model_dump(mode="json"),
+            timeout=10,
+        )
+        if not response.ok:
+            response.raise_for_status()
+
+    def _new_event_for_log(self, event_data: Any, level: int) -> Event:
+        """Create a new log event from arbitrary data"""
+        event_type = EventType.LOG
+        if level == logging.DEBUG:
+            event_type = EventType.LOG_DEBUG
+        elif level == logging.INFO:
+            event_type = EventType.LOG_INFO
+        elif level == logging.WARNING:
+            event_type = EventType.LOG_WARNING
+        elif level == logging.ERROR:
+            event_type = EventType.LOG_ERROR
+        elif level == logging.CRITICAL:
+            event_type = EventType.LOG_CRITICAL
+        return Event(
+            event_type=event_type,
+            event_data=event_data,
+        )

madsci/madsci_client/madsci/client/node/rest_node_client.py

@@ -54,7 +54,7 @@ def send_action(self, action_request: ActionRequest) -> ActionResult:
                 ("files", (file, Path(path).open("rb")))  # noqa: SIM115
                 for file, path in action_request.files.items()
             ]
-            print(files)
+            self.logger.log_debug(files)
 
             rest_response = requests.post(
                 f"{self.node.node_url}/action",

madsci/madsci_common/madsci/common/definition_loaders.py

@@ -3,59 +3,58 @@
 import argparse
 import json
 from pathlib import Path
-from typing import Any
+from typing import Any, Optional, Union
 
-from dotenv import load_dotenv
+from pydantic import AnyUrl
 
 from madsci.common.types.base_types import BaseModel
 from madsci.common.types.node_types import (
     NodeDefinition,
     NodeModuleDefinition,
     get_module_from_node_definition,
 )
-from madsci.common.types.squid_types import LabDefinition
+from madsci.common.types.squid_types import (
+    MANAGER_TYPE_DEFINITION_MAP,
+    LabDefinition,
+    ManagerDefinition,
+)
 from madsci.common.utils import search_for_file_pattern
 
 
 def madsci_definition_loader(
     model: type[BaseModel] = BaseModel,
     definition_file_pattern: str = "*.yaml",
     search_for_file: bool = True,
-) -> BaseModel:
+    return_all: bool = False,
+    cli_arg: Optional[str] = "definition",
+) -> Optional[Union[BaseModel, list[BaseModel]]]:
     """MADSci Definition Loader. Supports loading from a definition file, environment variables, and command line arguments, in reverse order of priority (i.e. command line arguments override environment variables, which override definition file values)."""
 
-    # Load environment variables from a .env file
-    load_dotenv()
-
-    parser = argparse.ArgumentParser(description="MADSci Definition Loader")
-    parser.add_argument(
-        "--definition",
-        type=Path,
-        help="The path to the MADSci configuration file.",
-    )
-    args, _ = parser.parse_known_args()
-    definition_file = args.definition
-    if not definition_file:
-        if not search_for_file:
-            raise ValueError(
-                "Definition file not specified, please specify a definition file using the --definition argument.",
-            )
-
-        # *Load from definition file
-        if search_for_file:
-            definition_files = search_for_file_pattern(
+    definition_files = []
+    if cli_arg:
+        parser = argparse.ArgumentParser(description="MADSci Definition Loader")
+        parser.add_argument(
+            f"--{cli_arg}",
+            type=Path,
+            help="The path to the MADSci configuration file.",
+        )
+        args, _ = parser.parse_known_args()
+        if args.definition:
+            definition_files.append(args.definition)
+
+    # *Load from definition file
+    if search_for_file:
+        definition_files.extend(
+            search_for_file_pattern(
                 definition_file_pattern,
                 parents=True,
                 children=True,
             )
-            if not definition_files:
-                raise ValueError(
-                    f"No definition files found matching pattern: {definition_file_pattern}. Please specify a valid configuration file path using the --definition argument.",
-                )
-
-        definition_file = definition_files[0]
+        )
 
-    return model.from_yaml(definition_file)
+    if return_all:
+        return [model.from_yaml(file) for file in definition_files]
+    return model.from_yaml(definition_files[0]) if definition_files else None
 
 
 def lab_definition_loader(
@@ -64,31 +63,11 @@ def lab_definition_loader(
     **kwargs: Any,
 ) -> LabDefinition:
     """Lab Definition Loader. Supports loading from a definition file, environment variables, and command line arguments, in reverse order of priority (i.e. command line arguments override environment variables, which override definition file values)."""
-    definition = madsci_definition_loader(
+    return madsci_definition_loader(
         model=model,
         definition_file_pattern=definition_file_pattern,
         **kwargs,
     )
-    for field_name, field in definition.lab_config.model_fields.items():
-        parser = argparse.ArgumentParser(
-            description=f"MADSci Lab Definition Loader for {field_name}",
-        )
-        parser.add_argument(
-            f"--{field_name}",
-            type=str,
-            help=f"[{field.annotation}] {field.description}",
-            default=None,
-        )
-    args, _ = parser.parse_known_args()
-    for field_name in definition.lab_config.model_fields:
-        if getattr(args, field_name) is not None:
-            setattr(
-                definition.lab_config,
-                field_name,
-                json.loads(getattr(args, field_name)),
-            )
-    definition.model_validate(definition)
-    return definition
 
 
 def node_definition_loader(
@@ -138,3 +117,45 @@ def node_definition_loader(
 
     # * Return the node and module definitions
     return node_definition, module_definition, config_values
+
+
+def manager_definition_loader(
+    model: type[BaseModel] = ManagerDefinition,
+    definition_file_pattern: str = "*.*manager.yaml",
+) -> ManagerDefinition:
+    """Loads all Manager Definitions available in the current context"""
+
+    # * Load from any standalone manager definition files
+    manager_definitions = madsci_definition_loader(
+        model=model,
+        definition_file_pattern=definition_file_pattern,
+        cli_arg=None,
+        search_for_file=True,
+        return_all=True,
+    )
+
+    # * Load from the lab manager's managers section
+    lab_manager_definition = lab_definition_loader(search_for_file=True)
+    if lab_manager_definition:
+        for manager in lab_manager_definition.managers.values():
+            if isinstance(manager, ManagerDefinition):
+                manager_definitions.append(manager)
+            elif isinstance(manager, AnyUrl):
+                # TODO: Support querying manager definition from URL, skip for now
+                pass
+            elif isinstance(manager, (Path, str)):
+                manager_definitions.append(ManagerDefinition.from_yaml(manager))
+
+    # * Upgrade to more specific manager types, where possible
+    refined_managers = []
+    for manager in manager_definitions:
+        if manager.manager_type in MANAGER_TYPE_DEFINITION_MAP:
+            refined_managers.append(
+                MANAGER_TYPE_DEFINITION_MAP[manager.manager_type].model_validate(
+                    manager
+                )
+            )
+        else:
+            refined_managers.append(manager)
+
+    return refined_managers

madsci/madsci_common/madsci/common/types/config_types.py

@@ -2,12 +2,12 @@
 
 from typing import Any, Optional
 
-from sqlmodel.main import Field
+from pydantic import Field
 
 from madsci.common.types.base_types import BaseModel
 
 
-class ConfigParameter(BaseModel, extra="allow"):
+class ConfigParameterDefinition(BaseModel, extra="allow"):
     """A configuration parameter definition for a MADSci system component."""
 
     name: str = Field(
@@ -31,6 +31,16 @@ class ConfigParameter(BaseModel, extra="allow"):
     )
     reset_on_change: bool = Field(
         title="Parameter Reset on Change",
-        description="Whether the node should restart whenever the parameter changes.",
+        description="Whether the configured object should restart/reset whenever the parameter changes.",
         default=True,
     )
+
+
+class ConfigParameterWithValue(ConfigParameterDefinition):
+    """A configuration parameter definition with value set"""
+
+    value: Optional[Any] = Field(
+        title="Parameter Value",
+        description="The value of the parameter, if set",
+        default=None,
+    )