From 25d6419098a926d66540c433224a92a5aa63f91c Mon Sep 17 00:00:00 2001 From: Aaron Kitzmiller Date: Tue, 9 Jan 2024 14:00:10 -0500 Subject: [PATCH 1/2] Prevent STARFISH_SERVER settings access when starfish plugin is not being used Filter out non-Harvard org tree orgs; results processing doesn't work when there is more than one org with the same name Use the Resultinator to group by error type. Missing code errors are not grouped so that printing the keys will display all the ones you care about. --- .../commands/id_import_new_allocations.py | 11 +- coldfront/plugins/ifx/calculator.py | 178 ++++++++++++------ coldfront/plugins/sftocf/utils.py | 5 +- 3 files changed, 135 insertions(+), 59 deletions(-) diff --git a/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py b/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py index 737898bd9..86864b7e7 100644 --- a/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py +++ b/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py @@ -20,11 +20,12 @@ ) from coldfront.core.utils.fasrc import update_csv, select_one_project_allocation, save_json from coldfront.core.resource.models import Resource -from coldfront.plugins.sftocf.utils import ( - StarFishRedash, - STARFISH_SERVER, - pull_sf_push_cf_redash -) +if ENV.bool('PLUGIN_SFTOCF', default=False): + from coldfront.plugins.sftocf.utils import ( + StarFishRedash, + STARFISH_SERVER, + pull_sf_push_cf_redash + ) from coldfront.plugins.fasrc.utils import ( AllTheThingsConn, match_entries_with_projects, diff --git a/coldfront/plugins/ifx/calculator.py b/coldfront/plugins/ifx/calculator.py index 9153e67e1..9b4df74f1 100644 --- a/coldfront/plugins/ifx/calculator.py +++ b/coldfront/plugins/ifx/calculator.py @@ -2,7 +2,8 @@ Custom billing calculator class for Coldfront ''' import logging -from collections import defaultdict +import re +from collections import defaultdict, OrderedDict from decimal import Decimal from django.core.exceptions import MultipleObjectsReturned from django.db import connection, transaction @@ -11,6 +12,7 @@ from django.conf import settings from ifxbilling.calculator import BasicBillingCalculator, NewBillingCalculator from ifxbilling.models import Account, Product, ProductUsage, Rate, BillingRecord +from ifxuser.models import Organization from coldfront.core.allocation.models import Allocation, AllocationStatusChoice from coldfront.plugins.ifx import adjust from .models import AllocationUserProductUsage @@ -25,6 +27,52 @@ class NewColdfrontBillingCalculator(NewBillingCalculator): OFFER_LETTER_TB_ATTRIBUTE = 'Offer Letter' OFFER_LETTER_CODE_ATTRIBUTE = 'Offer Letter Code' STORAGE_QUOTA_ATTRIBUTE = 'Storage Quota (TB)' + STORAGE_RESOURCE_TYPE = 'Storage' + + def calculate_billing_month(self, year, month, organizations=None, recalculate=False, verbosity=0): + ''' + Calculate a month of billing for the given year and month + + Returns a dict keyed by organization name that includes a count of successfully processed + product usages along with a list of error messages for each one that failed. + + Adjusts march 2023 due to bad DR issues + + :param year: Year that will be assigned to :class:`~ifxbilling.models.BillingRecord` objects + :type year: int + + :param month: Month that will be assigned to :class:`~ifxbilling.models.BillingRecord` objects + :type month: int + + :param organizations: List of specific organizations to process. If not set, all Harvard org_tree organizations will be processed. + :type organizations: list, optional + + :param recalculate: If set to True, will delete existing :class:`~ifxbilling.models.BillingRecord` objects + :type recalculate: bool, optional + + :param verbosity: Determines the amount of error reporting. Can be set to self.QUIET (no logger output), + self.CHATTY (use logger.error for errors), or self.LOUD (use logger.exception for errors). Defaults to QUIET. + :type verbosity: int, optional + + :return: dict keyed by organization name. Value is a dict of "successes" (a list of :class:`~ifxbilling.models.BillingRecord` objects) and + "errors" (a list of error messages) + :rtype: dict + ''' + self.verbosity = verbosity + + organizations_to_process = organizations + if not organizations_to_process: + organizations_to_process = Organization.objects.filter(org_tree='Harvard') + + results = {} + for organization in organizations_to_process: + result = self.generate_billing_records_for_organization(year, month, organization, recalculate) + results[organization.name] = result + + if year == 2023 and (month == 3 or month == 4): + adjust.march_april_2023_dr() + + return results def generate_billing_records_for_organization(self, year, month, organization, recalculate, **kwargs): ''' @@ -64,45 +112,54 @@ def generate_billing_records_for_organization(self, year, month, organization, r successes = [] errors = [] - projects = [po.project for po in organization.projectorganization_set.all()] - if not projects: - errors.append(f'No project found for {organization.name}') - else: - active = AllocationStatusChoice.objects.get(name='Active') - for project in projects: - for allocation in project.allocation_set.filter(status=active): - try: - allocation_tb = self.get_allocation_tb(allocation) - offer_letter_br, remaining_tb = self.process_offer_letter(year, month, organization, allocation, allocation_tb, recalculate) - if offer_letter_br: - successes.append(offer_letter_br) - if remaining_tb > Decimal('0'): - user_allocation_percentages = self.get_user_allocation_percentages(year, month, allocation) - for user_id, allocation_percentage_data in user_allocation_percentages.items(): + if organization.org_tree == 'Harvard': + projects = [po.project for po in organization.projectorganization_set.all()] + if not projects: + errors.append(f'No project found for {organization.name}') + else: + active = AllocationStatusChoice.objects.get(name='Active') + for project in projects: + for allocation in project.allocation_set.filter(status=active): + resources = allocation.resources.all() + if resources.count() == 1: + if resources[0].resource_type.name == self.STORAGE_RESOURCE_TYPE: try: - user = get_user_model().objects.get(id=user_id) - except get_user_model().DoesNotExist: - raise Exception(f'Cannot find user with id {user_id}') - brs = self.generate_billing_records_for_allocation_user( - year, - month, - user, - organization, - allocation, - allocation_percentage_data['fraction'], - allocation_tb, - recalculate, - remaining_tb, - ) - if brs: - successes.extend(brs) - except Exception as e: - errors.append(str(e)) - if self.verbosity == self.CHATTY: - logger.error(e) - if self.verbosity == self.LOUD: - logger.exception(e) - + allocation_tb = self.get_allocation_tb(allocation) + offer_letter_br, remaining_tb = self.process_offer_letter(year, month, organization, allocation, allocation_tb, recalculate) + if offer_letter_br: + successes.append(offer_letter_br) + if remaining_tb > Decimal('0'): + user_allocation_percentages = self.get_user_allocation_percentages(year, month, allocation) + for user_id, allocation_percentage_data in user_allocation_percentages.items(): + try: + user = get_user_model().objects.get(id=user_id) + except get_user_model().DoesNotExist: + raise Exception(f'Cannot find user with id {user_id}') + brs = self.generate_billing_records_for_allocation_user( + year, + month, + user, + organization, + allocation, + allocation_percentage_data['fraction'], + allocation_tb, + recalculate, + remaining_tb, + ) + if brs: + successes.extend(brs) + except Exception as e: + errors.append(str(e)) + if self.verbosity == self.CHATTY: + logger.error(e) + if self.verbosity == self.LOUD: + logger.exception(e) + else: + errors.append(f'Allocation {allocation} is not a storage allocation. Skipping.') + else: + errors.append(f'Allocation {allocation} has more than one resource.') + else: + errors.append(f'Organization {organization.slug} is not a Harvard organization.') return (successes, errors) def process_offer_letter(self, year, month, organization, allocation, allocation_tb, recalculate): @@ -643,16 +700,6 @@ def get_product_usage_for_allocation_user(self, year, month, user, organization, return aupu.product_usage - def calculate_billing_month(self, year, month, organizations=None, recalculate=False, verbosity=0): - ''' - Adjust march 2023 due to bad DR issues - ''' - results = super().calculate_billing_month(year, month, organizations, recalculate, verbosity) - if year == 2023 and (month == 3 or month == 4): - adjust.march_april_2023_dr() - - return results - class Resultinator(): @@ -665,9 +712,14 @@ def __init__(self, results): init ''' self.results = results - self.error_types = { - '' - } + self.error_types = OrderedDict([ + ('Not a Storage Allocation', r'^Allocation .*? is not a storage allocation. Skipping.'), + ('No AllocationProductUsage Found', r'^No AllocationUserProductUsage was found.*'), + # ('No Active User Account', r'^Unable to find an active user account record.*'), + ('Not a Harvard Organization', r'^Organization .*? is not a Harvard organization'), + ('Billing Record Exists for 0Tb usage', r'^Billing record already exists for usage 0.00 TB.*'), + ('Other', r'.*'), # Has to be last + ]) def get_errors_by_organization(self, organization_name=None): ''' @@ -697,5 +749,25 @@ def get_organizations_by_error_type(self): Returns a dictionary keyed by error type and listing the organization names with that issue. Errors that don't match are used as individual keys ''' - pass + # Set up the dict so that the named types are in order + errors_by_type = OrderedDict([('No project', [])]) + for k in self.error_types.keys(): + if k != 'Other': + errors_by_type[k] = [] + for lab, output in self.results.items(): + if output[1]: + if 'No project' in output[1][0]: + errors_by_type['No project'].append(lab) + else: + for error in output[1]: + for error_type, regex in self.error_types.items(): + # Other is the last one in the list and collects unmatched errors + if error_type == 'Other': + if error not in errors_by_type: + errors_by_type[error] = [] + errors_by_type[error].append(lab) + elif re.search(regex, error): + errors_by_type[error_type].append(lab) + break + return errors_by_type diff --git a/coldfront/plugins/sftocf/utils.py b/coldfront/plugins/sftocf/utils.py index 245a47b0d..d555d5f74 100644 --- a/coldfront/plugins/sftocf/utils.py +++ b/coldfront/plugins/sftocf/utils.py @@ -9,6 +9,7 @@ import requests from django.utils import timezone +from coldfront.config.env import ENV from coldfront.core.utils.common import import_from_settings from coldfront.core.utils.fasrc import ( read_json, @@ -33,7 +34,9 @@ datestr = datetime.today().strftime('%Y%m%d') logger = logging.getLogger('sftocf') -STARFISH_SERVER = import_from_settings('STARFISH_SERVER') +if ENV.bool('PLUGIN_SFTOCF', default=False): + STARFISH_SERVER = import_from_settings('STARFISH_SERVER') + svp = read_json('coldfront/plugins/sftocf/servers.json') From 91e54ebdaf951db86e9c419b1d5a7eef8f2c8966 Mon Sep 17 00:00:00 2001 From: Aaron Kitzmiller Date: Tue, 9 Jan 2024 14:04:11 -0500 Subject: [PATCH 2/2] make Resultinator output the default --- coldfront/plugins/ifx/calculator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/plugins/ifx/calculator.py b/coldfront/plugins/ifx/calculator.py index 9b4df74f1..d722f5097 100644 --- a/coldfront/plugins/ifx/calculator.py +++ b/coldfront/plugins/ifx/calculator.py @@ -72,7 +72,7 @@ def calculate_billing_month(self, year, month, organizations=None, recalculate=F if year == 2023 and (month == 3 or month == 4): adjust.march_april_2023_dr() - return results + return Resultinator(results) def generate_billing_records_for_organization(self, year, month, organization, recalculate, **kwargs): '''