Merge pull request #639 from smoy/google-workspace-users

Implement import mechanism for Google Workspace users

iambic/plugins/v0_1_0/google_workspace/handlers.py

@@ -11,6 +11,10 @@
     collect_project_groups,
     generate_group_templates,
 )
+from iambic.plugins.v0_1_0.google_workspace.user.template_generation import (
+    collect_project_users,
+    generate_user_templates,
+)
 
 if TYPE_CHECKING:
     from iambic.plugins.v0_1_0.google_workspace.iambic_plugin import (
@@ -43,6 +47,7 @@ async def import_google_resources(
         task_message = exe_message.copy()
         task_message.provider_id = workspace.project_id
         collector_tasks.append(collect_project_groups(task_message, config))
+        collector_tasks.append(collect_project_users(task_message, config))
 
     if collector_tasks:
         if base_runner and ctx.use_remote and remote_worker and not messages:
@@ -58,6 +63,7 @@ async def import_google_resources(
 
     if base_runner:
         generator_tasks = [
-            generate_group_templates(exe_message, config, base_output_dir)
+            generate_group_templates(exe_message, config, base_output_dir),
+            generate_user_templates(exe_message, config, base_output_dir),
         ]
         await asyncio.gather(*generator_tasks)

iambic/plugins/v0_1_0/google_workspace/tests/test_handlers.py

@@ -68,7 +68,13 @@ async def test_import_google_resources():
     ) as mock_collect_project_groups, unittest.mock.patch(
         "iambic.plugins.v0_1_0.google_workspace.handlers.generate_group_templates",
         new_callable=AsyncMock,
-    ) as mock_generate_group_templates:
+    ) as mock_generate_group_templates, unittest.mock.patch(
+        "iambic.plugins.v0_1_0.google_workspace.handlers.collect_project_users",
+        new_callable=AsyncMock,
+    ) as mock_collect_project_users, unittest.mock.patch(
+        "iambic.plugins.v0_1_0.google_workspace.handlers.generate_user_templates",
+        new_callable=AsyncMock,
+    ) as mock_generate_user_templates:
         # Call the import_google_resources function
         await import_google_resources(
             exe_message, config, base_output_dir, messages, remote_worker
@@ -79,3 +85,7 @@ async def test_import_google_resources():
         mock_generate_group_templates.assert_called_with(
             exe_message, config, base_output_dir
         )
+        mock_collect_project_users.assert_called()
+        mock_generate_user_templates.assert_called_with(
+            exe_message, config, base_output_dir
+        )

iambic/plugins/v0_1_0/google_workspace/user/models.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
+
+from pydantic import Field
+
+from iambic.core.iambic_enum import IambicManaged
+from iambic.core.models import BaseModel, BaseTemplate, ExpiryModel
+
+if TYPE_CHECKING:
+    from iambic.plugins.v0_1_0.google_workspace.iambic_plugin import GoogleProject
+
+
+GOOGLE_USER_TEMPLATE_TYPE = "NOQ::GoogleWorkspace::User"
+
+
+# https://developers.google.com/admin-sdk/directory/reference/rest/v1/users#UserName
+class WorkspaceUserName(BaseModel):
+    family_name: str = Field(
+        alias="familyName",
+        description="The user's last name. Required when creating a user account.",
+    )
+
+    given_name: str = Field(
+        alias="givenName",
+        description="The user's first name. Required when creating a user account.",
+    )
+
+    display_name: Optional[str] = Field(
+        alias="displayName",
+        description="The user's display name. Limit: 256 characters.",
+    )
+
+
+# https://developers.google.com/admin-sdk/directory/reference/rest/v1/users
+class WorkspaceUser(BaseModel, ExpiryModel):
+    primary_email: str = Field(
+        description="The user's primary email address. This property is required in a request to create a user account. The primaryEmail must be unique and cannot be an alias of another user.",
+    )
+    id: Optional[str] = Field(
+        None,
+        description="The unique ID for the user. A user id can be used as a user request URI's userKey.",
+    )
+    name: WorkspaceUserName = Field(
+        description="Holds the given and family names of the user, and the read-only fullName value. The maximum number of characters in the givenName and in the familyName values is 60. In addition, name values support unicode/UTF-8 characters, and can contain spaces, letters (a-z), numbers (0-9), dashes (-), forward slashes (/), and periods (.). For more information about character usage rules, see the administration help center. Maximum allowed data size for this field is 1KB.",
+    )
+
+    domain: str = Field(
+        description="this is not direct from user object from google response, but since user maps to a domain, we need to keep track of this information",
+    )
+
+    @property
+    def resource_type(self):
+        return "google:user"
+
+    @property
+    def resource_id(self):
+        return self.primary_email
+
+
+class GoogleWorkspaceUserTemplate(BaseTemplate, ExpiryModel):
+    template_type = GOOGLE_USER_TEMPLATE_TYPE
+    template_schema_url = (
+        "https://docs.iambic.org/reference/schemas/google_workspace_user_template"
+    )
+    # owner metadata seems strange for a user (maybe it makes more sense if its machine user)
+    # owner: Optional[str] = Field(None, description="Owner of the group")
+    properties: WorkspaceUser
+
+    @property
+    def resource_type(self):
+        return "google:user"
+
+    @property
+    def resource_id(self) -> str:
+        return self.properties.primary_email
+
+    @property
+    def default_file_path(self):
+        file_name = f"{self.properties.primary_email.split('@')[0]}.yaml"
+        return f"resources/google/users/{self.properties.domain}/{file_name}"
+
+    def _is_iambic_import_only(self, google_project: GoogleProject):
+        return (
+            google_project.iambic_managed == IambicManaged.IMPORT_ONLY
+            or self.iambic_managed == IambicManaged.IMPORT_ONLY
+        )
+
+
+# https://googleapis.github.io/google-api-python-client/docs/dyn/admin_directory_v1.users.html#list
+async def get_user_template(
+    service, user: dict, domain: str
+) -> GoogleWorkspaceUserTemplate:
+    # comment out because we don't have to make yet another network call
+    # members = await get_group_members(service, group)
+
+    file_name = f"{user['primaryEmail'].split('@')[0]}.yaml"
+    return GoogleWorkspaceUserTemplate(
+        file_path=f"resources/google/users/{domain}/{file_name}",
+        properties=dict(
+            domain=domain,
+            name=user["name"],
+            primary_email=user["primaryEmail"],
+        ),
+    )

iambic/plugins/v0_1_0/google_workspace/user/template_generation.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import asyncio
+import os
+from typing import TYPE_CHECKING
+
+from iambic.core.logger import log
+from iambic.core.models import ExecutionMessage
+from iambic.core.template_generation import (
+    create_or_update_template as common_create_or_update_template,
+)
+from iambic.core.template_generation import (
+    delete_orphaned_templates,
+    get_existing_template_map,
+)
+from iambic.plugins.v0_1_0.google_workspace.user.models import (
+    GOOGLE_USER_TEMPLATE_TYPE,
+    GoogleWorkspaceUserTemplate,
+)
+from iambic.plugins.v0_1_0.google_workspace.user.utils import list_users
+
+if TYPE_CHECKING:
+    from iambic.plugins.v0_1_0.google_workspace.iambic_plugin import (
+        GoogleProject,
+        GoogleWorkspaceConfig,
+    )
+
+
+def get_resource_dir_args(domain: str) -> list:
+    return ["user", domain]
+
+
+def get_response_dir(
+    exe_message: ExecutionMessage, google_project: GoogleProject, domain: str
+) -> str:
+    dir_args = get_resource_dir_args(domain)
+    dir_args.append("templates")
+    if exe_message.provider_id:
+        return exe_message.get_directory(*dir_args)
+    else:
+        return exe_message.get_directory(google_project.project_id, *dir_args)
+
+
+def get_user_dir(
+    base_dir: str,
+    domain: str,
+) -> str:
+    return str(
+        os.path.join(
+            base_dir, "resources", "google_workspace", *get_resource_dir_args(domain)
+        )
+    )
+
+
+def get_templated_resource_file_path(
+    resource_dir: str,
+    resource_email: str,
+) -> str:
+    unwanted_chars = ["}}_", "}}", ".", "-", " "]
+    resource_name = resource_email.split("@")[0].replace("{{", "").lower()
+    for unwanted_char in unwanted_chars:
+        resource_name = resource_name.replace(unwanted_char, "_")
+    return str(os.path.join(resource_dir, f"{resource_name}.yaml"))
+
+
+async def generate_domain_user_resource_files(
+    exe_message: ExecutionMessage, project: GoogleProject, domain: str
+):
+    account_user_response_dir = get_response_dir(exe_message, project, domain)
+
+    log.info("Caching Google users.", project_id=project.project_id, domain=domain)
+
+    users = await list_users(domain, project)
+    for user in users:
+        user.file_path = os.path.join(
+            account_user_response_dir,
+            f"{user.properties.primary_email.split('@')[0]}.yaml",
+        )
+        user.write()
+
+    log.info(
+        "Finished caching Google users.",
+        project_id=project.project_id,
+        domain=domain,
+        user_count=len(users),
+    )
+
+
+async def update_or_create_user_template(
+    discovered_user_template: GoogleWorkspaceUserTemplate,
+    existing_template_map: dict,
+    user_dir: str,
+) -> GoogleWorkspaceUserTemplate:
+    discovered_user_template.file_path = get_templated_resource_file_path(
+        user_dir,
+        discovered_user_template.properties.primary_email,
+    )
+
+    return common_create_or_update_template(
+        discovered_user_template.file_path,
+        existing_template_map,
+        discovered_user_template.resource_id,
+        GoogleWorkspaceUserTemplate,
+        {},
+        discovered_user_template.properties,
+        [],
+    )
+
+
+async def collect_project_users(
+    exe_message: ExecutionMessage, config: GoogleWorkspaceConfig
+):
+    assert exe_message.provider_id
+    project = config.get_workspace(exe_message.provider_id)
+    log.info("Beginning to retrieve Google users.", project=project.project_id)
+
+    await asyncio.gather(
+        *[
+            generate_domain_user_resource_files(exe_message, project, subject.domain)
+            for subject in project.subjects
+        ]
+    )
+
+    log.info("Finished retrieving Google user details", project=project.project_id)
+
+
+async def generate_user_templates(
+    exe_message: ExecutionMessage,
+    config: GoogleWorkspaceConfig,
+    output_dir: str,
+    detect_messages: list = None,
+):
+    """List all users in the domain"""
+
+    base_path = os.path.expanduser(output_dir)
+    existing_template_map = await get_existing_template_map(
+        base_path,
+        GOOGLE_USER_TEMPLATE_TYPE,
+        config.template_map,
+    )
+    all_resource_ids = set()
+
+    log.info("Updating and creating Google user templates.")
+
+    for workspace in config.workspaces:
+        for subject in workspace.subjects:
+            domain = subject.domain
+            user_dir = get_user_dir(base_path, domain)
+
+            users = await exe_message.get_sub_exe_files(
+                *get_resource_dir_args(domain),
+                "templates",
+                file_name_and_extension="**.yaml",
+            )
+            # Update or create templates
+            for user in users:
+                user = GoogleWorkspaceUserTemplate(file_path="unset", **user)
+                user.file_path = user.default_file_path
+                resource_template = await update_or_create_user_template(
+                    user, existing_template_map, user_dir
+                )
+                if not resource_template:
+                    # Template not updated. Most likely because it's an `enforced` template.
+                    continue
+                all_resource_ids.add(resource_template.resource_id)
+
+    # Delete templates that no longer exist
+    delete_orphaned_templates(list(existing_template_map.values()), all_resource_ids)
+
+    log.info("Finish updating and creating Google user templates.")

iambic/plugins/v0_1_0/google_workspace/user/utils.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from googleapiclient import _auth
+
+from iambic.core.logger import log
+from iambic.core.utils import aio_wrapper
+from iambic.plugins.v0_1_0.google_workspace.user.models import (
+    GoogleWorkspaceUserTemplate,
+)
+
+if TYPE_CHECKING:
+    from iambic.plugins.v0_1_0.google_workspace.iambic_plugin import GoogleProject
+
+
+# https://googleapis.github.io/google-api-python-client/docs/dyn/admin_directory_v1.users.html
+async def list_users(
+    domain: str, google_project: GoogleProject
+) -> list[GoogleWorkspaceUserTemplate]:
+    from iambic.plugins.v0_1_0.google_workspace.user.models import get_user_template
+
+    users = []
+    try:
+        service = await google_project.get_service_connection(
+            "admin", "directory_v1", domain
+        )
+        if not service:
+            return []
+        http = _auth.authorized_http(service._http.credentials)
+    except AttributeError as err:
+        log.exception("Unable to process google users.", error=err)
+        raise
+
+    req = await aio_wrapper(service.users().list, domain=domain)
+    while req is not None:
+        res = req.execute(http=http)
+        if res and "users" in res:
+            for user in res["users"]:
+                user_template = await get_user_template(service, user, domain)
+                users.append(user_template)
+
+        # handle pagination based on https://googleapis.github.io/google-api-python-client/docs/pagination.html
+        req = await aio_wrapper(service.users().list_next, req, res)
+    return users