From 686168243e008a94f603384c64f80cb270c2f89f Mon Sep 17 00:00:00 2001 From: claire-peters Date: Thu, 8 Feb 2024 14:20:28 -0500 Subject: [PATCH] update zone addition and removal conditions, improve logging --- .../allocation/allocation_detail.html | 4 +- .../management/commands/update_zones.py | 64 ++++++++++++------- coldfront/plugins/sftocf/utils.py | 12 ++-- 3 files changed, 47 insertions(+), 33 deletions(-) diff --git a/coldfront/core/allocation/templates/allocation/allocation_detail.html b/coldfront/core/allocation/templates/allocation/allocation_detail.html index 102bf154b..bcb015be7 100644 --- a/coldfront/core/allocation/templates/allocation/allocation_detail.html +++ b/coldfront/core/allocation/templates/allocation/allocation_detail.html @@ -26,8 +26,8 @@

Allocation Detail

- {% if allocation.project.sf_zone %} - View more information about this allocation on Starfish + {% if allocation.project.sf_zone and "tier" in allocation.get_parent_resource.name %} + View more information about your storage on Starfish {% endif %}
diff --git a/coldfront/plugins/sftocf/management/commands/update_zones.py b/coldfront/plugins/sftocf/management/commands/update_zones.py index 05798d9c6..1ff6cf026 100644 --- a/coldfront/plugins/sftocf/management/commands/update_zones.py +++ b/coldfront/plugins/sftocf/management/commands/update_zones.py @@ -32,13 +32,14 @@ def add_arguments(self, parser): def handle(self, *args, **options): dry_run = options['dry_run'] if dry_run: - print("DRY RUN") + print('DRY RUN') report = { 'dry_run': dry_run, 'deleted_zones': [], 'created_zones': [], 'allocations_missing_paths': [], + 'added_zone_ids': [], 'updated_zone_paths': [], 'updated_zone_groups': [] } @@ -47,24 +48,19 @@ def handle(self, *args, **options): starfish_zone_attr_type = ProjectAttributeType.objects.get(name='Starfish Zone') # collect all projects that have active allocations on Starfish projects_with_allocations = Project.objects.filter( + status__name='Active', allocation__status__name='Active', allocation__resources__in=sf.get_corresponding_coldfront_resources(), + title__in=sf.get_groups() # confirm the projects have groups in Starfish ).distinct() - # also confirm that the projects are available as groups in Starfish - projects_with_allocations = projects_with_allocations.filter( - title__in=sf.get_groups(), - ) - projects_with_zones = projects_with_allocations.filter( projectattribute__proj_attr_type=starfish_zone_attr_type, ) # for the projects that do have zones, ensure that its zone: sf_cf_vols = sf.get_volumes_in_coldfront() for project in projects_with_zones: - zone_id = project.projectattribute_set.get( - proj_attr_type__name='Starfish Zone', - ).value + zone_id = project.sf_zone zone = sf.get_zones(zone_id) # has all the allocation paths associated with the project @@ -85,16 +81,6 @@ def handle(self, *args, **options): if missing_paths: continue paths = [f'{a.resources.first().name.split("/")[0]}:{a.path}' for a in storage_allocations] + zone_paths_not_in_cf - # delete any zones that have no paths - if not paths: - if not dry_run: - sf.delete_zone(zone['id']) - # delete projectattribute - project.projectattribute_set.get( - proj_attr_type=starfish_zone_attr_type, - ).delete() - report['deleted_zones'].append(zone['name']) - continue if not set(paths) == set(zone['paths']): if not dry_run: update_paths['paths'] = paths @@ -107,7 +93,7 @@ def handle(self, *args, **options): # has the project AD group in “managing_groups” update_groups = zone['managing_groups'] - zone_group_names = [g['groupname'] for g in zone['managing_groups']] + zone_group_names = [g['groupname'] for g in update_groups] if project.title not in zone_group_names: if not dry_run: update_groups.append({'groupname': project.title}) @@ -131,16 +117,46 @@ def handle(self, *args, **options): if e.response.status_code == 409: logger.warning('zone for %s already exists; adding zoneid to Project and breaking', project.title) zone = sf.get_zone_by_name(project.title) + report['added_zone_ids'].append([project.title, zone['id']]) + elif e.response.status_code == 402: + logger.error('zone quota reached; can no longer add any zones.') + continue else: - logger.error( - 'unclear error prevented creation of zone for project %s. error: %s', - project.title, e - ) + err = f'unclear error prevented creation of zone for project {project.title}. error: {e.response}' + logger.error(err) + print(err) + continue + except ValueError as e: + err = f"error encountered. If no groups returned, LDAP group doesn't exist: {e}, {project.title}" + logger.error(err) + print(err) + continue project.projectattribute_set.get_or_create( proj_attr_type=starfish_zone_attr_type, value=zone['id'], ) else: report['created_zones'].append(project.title) + + # check whether to delete zones of projects with no active SF storage allocations + potential_delete_zone_attr_projs = Project.objects.filter( + projectattribute__proj_attr_type__name='Starfish Zone' + ).exclude( + title__in=[p.title for p in projects_with_allocations] + ) + for project in potential_delete_zone_attr_projs: + print(project, project.pk) + zone = sf.get_zones(project.sf_zone) + zone_paths_not_in_cf = [p for p in zone['paths'] if p.split(':')[0] not in sf_cf_vols] + # delete any zones that have no paths + if not zone_paths_not_in_cf: + if not dry_run: + sf.delete_zone(zone['id']) + # delete projectattribute + project.projectattribute_set.get( + proj_attr_type=starfish_zone_attr_type, + ).delete() + report['deleted_zones'].append(zone['name']) + continue print(report) logger.warning(report) diff --git a/coldfront/plugins/sftocf/utils.py b/coldfront/plugins/sftocf/utils.py index 08fa6fc5f..e583b1ee0 100644 --- a/coldfront/plugins/sftocf/utils.py +++ b/coldfront/plugins/sftocf/utils.py @@ -133,6 +133,7 @@ def add_zone_group_to_ad(group_name): error = f'Error adding {group_name} to starfish_users: {e}' print(error) logger.warning(error) + raise class StarFishServer: @@ -192,14 +193,11 @@ def get_zone_by_name(self, zone_name): def create_zone(self, zone_name, paths, managers, managing_groups): """Create a zone via the API""" url = self.api_url + 'zone/' - zone_paths = paths if paths else [] - zone_managers = managers if managers else [] - zone_managing_groups = managing_groups if managing_groups else [] data = { "name": zone_name, - "paths": zone_paths, - "managers": zone_managers, - "managing_groups": zone_managing_groups, + "paths": paths, + "managers": managers, + "managing_groups": managing_groups, } logger.debug(data) response = return_post_json(url, data=data, headers=self.headers) @@ -243,7 +241,7 @@ def zone_from_project(self, project_obj): managers = [f'{project_obj.pi.username}'] managing_groups = [{'groupname': project_obj.title}] add_zone_group_to_ad(project_obj.title) - return self.create_zone(zone_name, paths, managers, managing_groups) + return self.create_zone(zone_name, paths, [], managing_groups) def get_corresponding_coldfront_resources(self): resources = Resource.objects.filter(