Automated starfish zone management (#275)

* add zone interaction methods (get, create, update, delete) to Starfish API wrapper

* add zone_report function for checking basic zone characteristics on the sf server

* add rest api to pull_resource_data, add project_obj-based zone addition method

* add method for group member addition to ldap and add that to the zone creation routine

* add ad group addition routine to zone update method

* add allocation_to_zone function

* add get_attribute and get_attribute_list to project model

* link addition to gui

coldfront/core/allocation/templates/allocation/allocation_detail.html

@@ -26,6 +26,9 @@
 <div class="mb-3">
   <h2>Allocation Detail</h2>
   <button class="btn btn-primary" id="download"> Download PDF </button>
+  {% if allocation.project.sf_zone and "tier" in allocation.get_parent_resource.name %}
+  <a class="btn btn-success" href="https://starfish.rc.fas.harvard.edu/#/browser?zone={{allocation.project.sf_zone}}" role="button" id="starfish"> View more information about your storage on Starfish </a>
+  {% endif %}
   <hr>
 </div>
 

coldfront/core/department/models.py

@@ -47,11 +47,9 @@ class Department(Organization):
     objects = DepartmentSelector()
     history = HistoricalRecords()
 
-
     class Meta:
         proxy = True
 
-
     def get_projects(self):
         """Get all projects related to the Department, either directly or indirectly.
         """

coldfront/core/department/templates/department/department_detail.html

@@ -23,10 +23,10 @@
     {{ form.non_field_errors }}
   </div>
 {% endif %}
-<div id = "invoice">
+<div id="invoice">
 <div class="mb-3" >
     <h2>FAS Research Computing Usage Report</h2>
-    <button class = "btn btn-primary" id = "download"> Download PDF </button>
+    <button class="btn btn-primary" id="download"> Download PDF </button>
 
     <hr>
   </div>

coldfront/core/project/management/commands/add_default_project_choices.py

@@ -28,9 +28,10 @@ def handle(self, *args, **options):
             AttributeType.objects.get_or_create(name=attribute_type)
 
         for name, attribute_type, has_usage, is_private in (
+            ('Starfish Zone', 'Int', False, False),
             # UBCCR defaults
-            ('Project ID', 'Text', False, False),
-            ('Account Number', 'Int', False, True),
+            # ('Project ID', 'Text', False, False),
+            # ('Account Number', 'Int', False, True),
         ):
             ProjectAttributeType.objects.update_or_create(
                 name=name,

coldfront/core/project/models.py

@@ -135,6 +135,10 @@ def latest_publication(self):
             return self.publication_set.order_by('-created')[0]
         return None
 
+    @property
+    def sf_zone(self):
+        return self.get_attribute('Starfish Zone')
+
     @property
     def needs_review(self):
         """
@@ -215,12 +219,37 @@ def has_perm(self, user, perm):
         perms = self.user_permissions(user)
         return perm in perms
 
+    def get_attribute(self, name):
+        """
+        Params:
+            name (str): name of the project attribute type
+
+        Returns:
+            str: value of the first attribute found for this project with the specified name
+        """
+        attr = self.projectattribute_set.filter(proj_attr_type__name=name).first()
+        if attr:
+            return attr.value
+        return None
+
+    def get_attribute_list(self, name):
+        """
+        Params:
+            name (str): name of the project attribute type
+
+        Returns:
+            list: the list of values of the attributes found with specified name
+        """
+        attr = self.projectattribute_set.filter(proj_attr_type__name=name)
+        return [a.value for a in attr]
+
     def __str__(self):
         return self.title
 
     def natural_key(self):
         return (self.title,) + self.pi.natural_key()
 
+
 class ProjectAdminComment(TimeStampedModel):
     """ A project admin comment is a comment that an admin can make on a project.
 

coldfront/core/project/templates/project/project_detail.html

@@ -92,7 +92,11 @@ <h3 class="d-inline"><i class="fas fa-list" aria-hidden="true"></i> Project Allo
     <div class="table-responsive">
       {% if storage_allocations %}
         <table id="invoice_table" class="table table-hover">
-        <h4>Storage</h4>
+        <h4>Storage &nbsp;
+	{% if project.sf_zone %}
+	  <a class="btn btn-success" href="https://starfish.rc.fas.harvard.edu/#/browser?zone={{project.sf_zone}}" role="button"> View detailed storage allocation information on Starfish </a>
+	{% endif %}
+	</h4>
         <thead>
           <tr>
             <th scope="col">Resource Name</th>

coldfront/core/project/views.py

@@ -670,7 +670,7 @@ def post(self, request, *args, **kwargs):
 
                     if 'coldfront.plugins.ldap' in settings.INSTALLED_APPS:
                         try:
-                            ldap_conn.add_member_to_group(
+                            ldap_conn.add_user_to_group(
                                 user_obj.username, project_obj.title,
                             )
                             logger.info(

coldfront/core/utils/common.py

@@ -36,6 +36,11 @@ def write(self, value):
         """Write the value by returning it, instead of storing in a buffer."""
         return value
 
+def uniques_and_intersection(list1, list2):
+    intersection = list(set(list1) & set(list2))
+    list1_unique = list(set(list1) - set(list2))
+    list2_unique = list(set(list2) - set(list1))
+    return (list1_unique, intersection, list2_unique)
 
 def su_login_callback(user):
     """Only superusers are allowed to login as other users

coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py

@@ -56,7 +56,7 @@ def handle(self, *args, **options):
         save_json(result_file, resp_json_by_lab)
 
         # Remove allocations for labs not in Coldfront, add those labs to a list
-        result_json_cleaned, proj_models = match_entries_with_projects(resp_json_by_lab)
+        result_cleaned, proj_models = match_entries_with_projects(resp_json_by_lab)
 
         redash_api = StarFishRedash()
         allocation_usages = redash_api.return_query_results(query='subdirectory')
@@ -67,7 +67,7 @@ def handle(self, *args, **options):
                 project.status = ProjectStatusChoice.objects.get(name='Active')
                 project.save()
 
-        for lab, allocations in result_json_cleaned.items():
+        for lab, allocations in result_cleaned.items():
             project = proj_models.get(title=lab)
             for entry in allocations:
                 lab_name = entry['lab']

coldfront/plugins/fasrc/utils.py

@@ -31,9 +31,9 @@ def produce_query_statement(self, vol_type, volumes=None):
                 'match': "(e:Quota) MATCH (d:ConfigValue {Name: 'Quota.Invocation'})",
                 'server': 'filesystem',
                 'validation_query':
-                        "NOT ((e.SizeGB IS null) OR (e.usedBytes = 0 AND e.SizeGB = 1024)) \
-                        AND (datetime() - duration('P31D') <= datetime(r.DotsLFSUpdateDate)) \
-                        AND NOT (e.Path IS null)",
+                    "NOT ((e.SizeGB IS null) OR (e.usedBytes = 0 AND e.SizeGB = 1024)) \
+                    AND (datetime() - duration('P31D') <= datetime(r.DotsLFSUpdateDate)) \
+                    AND NOT (e.Path IS null)",
                 'r_updated': 'DotsLFSUpdateDate',
                 'storage_type': 'Quota',
                 'usedgb': 'usedGB',
@@ -49,10 +49,10 @@ def produce_query_statement(self, vol_type, volumes=None):
                 'match': "(e:IsilonPath) MATCH (d:ConfigValue {Name: 'IsilonPath.Invocation'})",
                 'server': 'Isilon',
                 'validation_query': "r.DotsUpdateDate = d.DotsUpdateDate \
-                        AND NOT (e.Path =~ '.*/rc_admin/.*')\
-                        AND (e.Path =~ '.*labs.*')\
-                        AND (datetime() - duration('P31D') <= datetime(r.DotsUpdateDate)) \
-                        AND NOT (e.SizeGB = 0)",
+                    AND NOT (e.Path =~ '.*/rc_admin/.*')\
+                    AND (e.Path =~ '.*labs.*')\
+                    AND (datetime() - duration('P31D') <= datetime(r.DotsUpdateDate)) \
+                    AND NOT (e.SizeGB = 0)",
                 'fs_path':'Path',
                 'r_updated': 'DotsUpdateDate',
                 'storage_type': 'Isilon',
@@ -253,6 +253,7 @@ def pair_allocations_data(project, quota_dicts):
         dicts = [
             d for d in quota_dicts
             if d['fs_path'] and allocation.path.lower() == d['fs_path'].lower()
+            and d['server'] in allocation.resources.first().name
         ]
         if dicts:
             log_message = f'Path-based match: {allocation}, {allocation.path}, {dicts[0]}'

coldfront/plugins/ldap/management/commands/id_add_new_projects.py

@@ -45,7 +45,8 @@ def handle(self, *args, **kwargs):
                         'sAMAccountName': groups,
                         'managedBy': '*'
                         }, attributes=['sAMAccountName'])
-        ad_group_names = [group['sAMAccountName'][0] for group in ad_groups] # get all AD group names
+        # get all AD group names
+        ad_group_names = [group['sAMAccountName'][0] for group in ad_groups]
         # remove AD groups that already have a corresponding ColdFront project
         ad_only = list(set(ad_group_names) - set(project_titles))
         errortracker = {
@@ -72,7 +73,7 @@ def handle(self, *args, **kwargs):
         added_projects, errortracker = add_new_projects(groupusercollections, errortracker)
         print(f"added {len(added_projects)} projects: ", [a[0] for a in added_projects])
         print("errs: ", errortracker)
-        logger.warning(errortracker)
+        logger.warning("errors: %s", errortracker)
         not_added = [
             {'title': i, 'info': k} for k, v in errortracker.items() for i in v
         ]

coldfront/plugins/ldap/utils.py

@@ -12,7 +12,9 @@
 from ldap3.extend.microsoft.addMembersToGroups import ad_add_members_to_groups
 from ldap3.extend.microsoft.removeMembersFromGroups import ad_remove_members_from_groups
 
-from coldfront.core.utils.common import import_from_settings
+from coldfront.core.utils.common import (
+    import_from_settings, uniques_and_intersection
+)
 from coldfront.core.field_of_science.models import FieldOfScience
 from coldfront.core.utils.fasrc import (
     id_present_missing_users,
@@ -32,8 +34,7 @@
 username_ignore_list = import_from_settings('username_ignore_list', [])
 
 class LDAPConn:
-    """
-    LDAP connection object
+    """LDAP connection object
     """
     def __init__(self, test=False):
 
@@ -159,35 +160,34 @@ def return_group_by_name(self, groupname, return_as='dict'):
             raise ValueError("no groups returned")
         return group[0]
 
-    def add_member_to_group(self, user_name, group_name):
-        # get group
+    def add_user_to_group(self, user_name, group_name):
         group = self.return_group_by_name(group_name)
-        # get user
-        try:
-            user = self.return_user_by_name(user_name)
-        except ValueError as e:
-            raise e
+        user = self.return_user_by_name(user_name)
+        self.add_member_to_group(user, group)
+
+    def add_group_to_group(self, group_name, parent_group_name):
+        group = self.return_group_by_name(group_name)
+        parent_group = self.return_group_by_name(parent_group_name)
+        self.add_member_to_group(group, parent_group)
+
+    def add_member_to_group(self, member, group):
         group_dn = group['distinguishedName']
-        user_dn = user['distinguishedName']
+        member_dn = member['distinguishedName']
         try:
-            result = ad_add_members_to_groups(self.conn, [user_dn], group_dn, fix=True)
+            result = ad_add_members_to_groups(
+                self.conn, [member_dn], group_dn, fix=True)
         except Exception as e:
             raise e
         return result
 
     def remove_member_from_group(self, user_name, group_name):
         # get group
-        try:
-            group = self.return_group_by_name(group_name)
-        except ValueError as e:
-            raise e
+        group = self.return_group_by_name(group_name)
         # get user
-        try:
-            user = self.return_user_by_name(user_name)
-        except ValueError as e:
-            raise e
+        user = self.return_user_by_name(user_name)
         if user['gidNumber'] == group['gidNumber']:
-            raise ValueError("group is user's primary group - please contact FASRC support to remove this user from your group.")
+            raise ValueError(
+                "Group is user's primary group. Please contact FASRC support to remove this user from your group.")
         group_dn = group['distinguishedName']
         user_dn = user['distinguishedName']
         try:
@@ -327,13 +327,6 @@ def format_template_assertions(attr_search_dict, search_operator='and'):
         search_filter = f'({match_operator[search_operator]}'+search_filter+')'
     return search_filter
 
-def uniques_and_intersection(list1, list2):
-    intersection = list(set(list1) & set(list2))
-    list1_unique = list(set(list1) - set(list2))
-    list2_unique = list(set(list2) - set(list1))
-    return (list1_unique, intersection, list2_unique)
-
-
 def is_string(value):
     return isinstance(value, str)
 
@@ -348,7 +341,7 @@ def sort_dict_on_conditional(dict1, condition):
 def cleaned_membership_query(proj_membs_mans):
     search_errors, proj_membs_mans = sort_dict_on_conditional(proj_membs_mans, is_string)
     if search_errors:
-        logger.error('could not return members and manager for some groups:\n%s',
+        logger.error('could not return members and manager for some groups: %s',
                         search_errors)
     return proj_membs_mans, search_errors