Skip to content

Commit

Permalink
Merge pull request #269 from fasrc/cp_slurm_fixes
Browse files Browse the repository at this point in the history 
slurm adjustments
  • Loading branch information
@claire-peters
claire-peters authored Dec 12, 2023
2 parents fa83246 + 318416c commit b9113ab
Show file tree
Hide file tree
Showing 21 changed files with 172 additions and 843 deletions.
14 changes: 6 additions & 8 deletions coldfront/core/department/templates/department/department_detail.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,12 +89,12 @@ <h3><i class="fas fa-list" aria-hidden="true"></i> Usage Summary For: {{ departm

{% comment %} <td style="background-color:#D3D3D3" id = "totalamount"></td> {% endcomment %}
</tr>
<tr>
<!-- <tr>
<th style="background-color:#D3D3D3" bordercolor="red" scope="row" class="text-nowrap">Total Amount Due, Quarterly Compute: </th>
<td style="background-color:#D3D3D3">${{ compute_full_price|floatformat:2 }}</td>
{% comment %} <td style="background-color:#D3D3D3" id = "totalamount"></td> {% endcomment %}
</tr>
</tr> -->
{% else %}
<tr>
<th scope="row" class="text-nowrap">Your Labs:</th>
Expand All @@ -113,7 +113,7 @@ <h3><i class="fas fa-list" aria-hidden="true"></i> Usage Summary For: {{ departm
<td>{{ allocation_users.count }}</td>
</tr>
<tr>
<th style="background-color:#D3D3D3" bordercolor="red" scope="row" class="text-nowrap">Amount Due for Projects: </th>
<th style="background-color:#D3D3D3" bordercolor="red" scope="row" class="text-nowrap">Total Amount Due, Monthly Storage:</th>
<td style="background-color:#D3D3D3">${{ storage_full_price|floatformat:2 }}</td>

{% comment %} <td style="background-color:#D3D3D3" id = "totalamount"></td> {% endcomment %}
Expand Down Expand Up @@ -224,7 +224,7 @@ <h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i>Monthly Stor
<div class="html2pdf__page-break"></div>
<div class="card mb-3">
<div class="card-header">
<h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i>Quarterly Compute Invoice Details</h3>
<h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i>Quarterly Cluster Usage Details</h3>
<span class="badge badge-secondary">{{compute_pi_dict.keys|length}}</span>
</div>
<div class="card-body">
Expand All @@ -238,7 +238,7 @@ <h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i>Quarterly Co
<th scope="col">Allocation</th>
<th scope="col">Users</th>
<th scope="col">Quota (TB)</th>
<th scope="col">Quarterly Cost to Date</th>
<!-- <th scope="col">Quarterly Cost to Date</th> -->
</tr>
</thead>
<tbody>
Expand All @@ -256,7 +256,6 @@ <h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i>Quarterly Co
</td>
</tr>
{% for allocation in allocs %}
{% if allocation.resources.first.resource_type.name == "Cluster" %}
<tr style="background-color:#FFFFFF" class="child-allocation">
<td data-search="{{ pi.full_name }}" data-sort="{{ pi.full_name }}"></td>
<td data-search="{{allocation.project.title}}">
Expand All @@ -274,9 +273,8 @@ <h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i>Quarterly Co
</td>
<td>{{ allocation.allocationuser_set.count }}</td>
<td>{{ allocation.size|floatformat:1 }}</td>
<td>${{ allocation.cost|floatformat:2 }}</td>
<!-- <td>${{ allocation.cost|floatformat:2 }}</td> -->
</tr>
{% endif %}
{% endfor %}
{% endfor %}
</tbody>
Expand Down
2 changes: 2 additions & 0 deletions coldfront/core/utils/management/commands/add_scheduled_tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ def handle(self, *args, **options):
'fasrc': ['import_quotas', 'id_import_allocations'],
'sftocf': ['pullsf_pushcf_redash', 'pull_resource_data'],
'ldap': ['update_group_membership_ldap', 'id_add_projects'],
'slurm': ['slurm_sync'],
'xdmod': ['xdmod_usage'],
}
scheduled = [task.func for task in Schedule.objects.all()]

Expand Down
10 changes: 1 addition & 9 deletions coldfront/core/utils/management/commands/create_superuser.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
import os

from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
base_dir = settings.BASE_DIR

class Command(BaseCommand):
Expand All @@ -14,7 +10,3 @@ def handle(self, *args, **options):
admin_user.is_superuser = True
admin_user.is_staff = True
admin_user.save()

for user in get_user_model().objects.all():
user.set_password('test1234')
user.save()
3 changes: 1 addition & 2 deletions coldfront/core/utils/management/commands/import_add_allocations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,7 @@
from django.core.exceptions import ValidationError, MultipleObjectsReturned
from django.core.management.base import BaseCommand

from coldfront.core.allocation.models import (Allocation,
AllocationUser,
from coldfront.core.allocation.models import (AllocationUser,
AllocationAttribute,
AllocationAttributeType,
AllocationStatusChoice,
Expand Down
24 changes: 4 additions & 20 deletions coldfront/core/utils/management/commands/import_allocationattribute.py
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,12 @@
import datetime
import os
import json

from dateutil.relativedelta import relativedelta
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import BaseCommand

from coldfront.core.allocation.models import (Allocation, AllocationAttribute,
AllocationAttributeType,
AllocationStatusChoice,
AllocationUser,
AllocationUserStatusChoice)
from coldfront.core.field_of_science.models import FieldOfScience
from coldfront.core.grant.models import (Grant, GrantFundingAgency,
GrantStatusChoice)
from coldfront.core.project.models import (Project, ProjectStatusChoice,
ProjectUser, ProjectUserRoleChoice,
ProjectUserStatusChoice)
from coldfront.core.publication.models import Publication, PublicationSource
from coldfront.core.resource.models import (Resource, ResourceAttribute,
ResourceAttributeType,
ResourceType)
from coldfront.core.user.models import UserProfile
AllocationStatusChoice)
from coldfront.core.resource.models import (Resource,
ResourceAttribute,
ResourceAttributeType)

base_dir = settings.BASE_DIR

Expand Down
29 changes: 11 additions & 18 deletions coldfront/core/utils/management/commands/import_allocationquotas.py
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,13 @@
import csv
import datetime
import os
import json
import logging

from dateutil.relativedelta import relativedelta
from django.conf import settings
from django.contrib.auth import get_user, get_user_model
from django.core.management import call_command
from django.core.management.base import BaseCommand

from coldfront.core.allocation.models import (Allocation, AllocationAttribute,
AllocationAttributeType)
from coldfront.core.project.models import (Project)
from coldfront.core.resource.models import (Resource, )
from coldfront.config.env import ENV

import pandas as pd
Expand All @@ -38,14 +32,14 @@ def add_arguments(self, parser):
default='tier0',
help='Storage tier',
)

def handle(self, *args, **options):

LOCALDATA_ROOT = ENV.str('LOCALDATA_ROOT', default=base_dir)
storage = options['storage']
storage = options['storage']
print(storage)
fileName = storage + "_allocation.csv"
tier = options['tier']
tier = options['tier']
resource_name = storage + '/' + tier
print("Loading data for: " + resource_name)
lab_list_file = os.path.join(LOCALDATA_ROOT, 'local_data/',fileName)
Expand All @@ -66,14 +60,14 @@ def handle(self, *args, **options):
try:
filtered_query = Project.objects.get(title = lab_name) # find project
allocations = Allocation.objects.filter(project = filtered_query, resources__name=resource_name, status__name='Active')
if(allocations.count() == 0):
if allocations.count() == 0:
print("Allocation not found:" + lab_name + ": "+resource_name)
tocsv = [lab_name,resource_name,"Allocation"]
writer.writerow(tocsv)
writer.writerow(tocsv)
continue

allocation= allocations[0]
if (allocation): # get allocation
if allocation: # get allocation
allocation_attribute_type_obj = AllocationAttributeType.objects.get(
name='Storage Quota (TB)')
try:
Expand All @@ -87,28 +81,27 @@ def handle(self, *args, **options):
except AllocationAttribute.DoesNotExist:
allocation_attribute_exist = False

if (not allocation_attribute_exist):
if not allocation_attribute_exist:
allocation_attribute_obj,_ =AllocationAttribute.objects.get_or_create(
allocation_attribute_type=allocation_attribute_type_obj,
allocation=allocation,
value = lab_allocation)
allocation_attribute_type_obj.save()


allocation_attribute_obj.allocationattributeusage.value = lab_usage
allocation_attribute_obj.allocationattributeusage.save()

allocation_attribute_type_payment = AllocationAttributeType.objects.get(
name='RequiresPayment')
allocation_attribute_payment, _ = AllocationAttribute.objects.get_or_create(
allocation_attribute_type=allocation_attribute_type_payment,
allocation=allocation,
value=True)
value=True)
allocation_attribute_payment.save()
except Project.DoesNotExist:
print("Project not found: " + lab_name)
tocsv = [lab_name,resource_name,"Project"]
writer.writerow(tocsv)
writer.writerow(tocsv)
continue
proj_allocation.close()

100 changes: 32 additions & 68 deletions coldfront/core/utils/management/commands/import_allocationuser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,20 +87,6 @@ def handle(self, *args, **options):
with open(lab_data) as f:
data = json.load(f)

# if (not filtered_query): # if not found project, then create project
# project_obj, _ = Project.objects.get_or_create(
# #pi = pi1,
# title = lab_name,
# description= lab_name + ' storage allocation',
# field_of_science=FieldOfScience.objects.get(
# #FIXME: Description of the DEPT
# description='Other'),
# status=ProjectStatusChoice.objects.get(name='Active'),
# force_review=True
# )
# start_date = datetime.datetime.now()
# end_date = datetime.datetime.now() + relativedelta(days=365)

# else: # found project
allocations = Allocation.objects.filter(project=filtered_query, resources__name=resource_name, status__name='Active')
if allocations.count() == 0:
Expand Down Expand Up @@ -140,22 +126,17 @@ def handle(self, *args, **options):
lab_usage_in_tb = round(lab_usage_in_tb, 2)
# lab_usage_in_tb_str = str(lab_usage_in_tb)
allocation= allocations[0]
if (allocation): # get allocation
if allocation: # get allocation
allocation_attribute_type_obj = AllocationAttributeType.objects.get(
name='Storage Quota (TB)')
try:
allocation_attribute_obj = AllocationAttribute.objects.get(
allocation_attribute_type=allocation_attribute_type_obj,
allocation=allocation,
)
allocation_attribute_obj.value = lab_allocation_in_tb_str
allocation_attribute_obj.save()
allocation_attribute_exist = True
except AllocationAttribute.DoesNotExist:
allocation_attribute_exist = False

if (not allocation_attribute_exist):
allocation_attribute_obj,_ =AllocationAttribute.objects.get_or_create(
allocation_attribute_obj, created = AllocationAttribute.objects.get_or_create(
allocation_attribute_type=allocation_attribute_type_obj,
allocation=allocation,
defaults={'value': lab_allocation_in_tb_str}
)

if created:
allocation_attribute_obj, _ = AllocationAttribute.objects.get_or_create(
allocation_attribute_type=allocation_attribute_type_obj,
allocation=allocation,
value = lab_allocation_in_tb_str)
Expand All @@ -175,9 +156,9 @@ def handle(self, *args, **options):
# checking my user_json_dictinary
# loop through my allocation_users set
for allocation_user in allocation_users:
allocation_user_username = (allocation_user.user.username)
allocation_user_username = allocation_user.user.username
user_obj = get_user_model().objects.get(username=allocation_user_username)
if allocation_user_username in user_json_dict:
user_obj = get_user_model().objects.get(username = allocation_user_username)
allocationuser_obj = AllocationUser.objects.get(user=user_obj)
allocationuser_obj.status = AllocationUserStatusChoice.objects.get(name='Active')
one_user_logical_usage = user_json_dict[allocation_user_username]['logical_usage']
Expand All @@ -189,20 +170,16 @@ def handle(self, *args, **options):
allocation.save()
user_json_dict.pop(allocation_user_username)
else:
try:
user_obj = get_user_model().objects.get(username = allocation_user_username)
allocationuser_obj = AllocationUser.objects.get(user=user_obj)
allocationuser_obj.status = AllocationUserStatusChoice.objects.get(name='Removed')
allocationuser_obj.usage = 0
allocationuser_obj.usage_bytes = 0
allocationuser_obj.unit = ''
allocationuser_obj.allocation_group_usage_bytes = lab_data["kbytes"]
allocationuser_obj.allocation_group_quota = lab_data["quota"]
allocationuser_obj.save()
allocation.save()
except Exception as e:
print(f'Error: {e}')
# allocation_users.remove(allocation_user) # remove this particular allocation_user
allocationuser_obj, _ = AllocationUser.objects.update_or_create(
user=user_obj,
allocation=allocation,
defaults={
'status': AllocationUserStatusChoice.objects.get(name='Removed'),
'usage': 0,
'usage_bytes': 0,
'unit': '',
}
)

for json_user in user_json_dict:
try:
Expand All @@ -228,31 +205,18 @@ def handle(self, *args, **options):
# )
# get_user_model().objects.get(username=json_user).save()

# raise Exception(f'Cannot find user {json_user}')

try:
allocationuser_obj = AllocationUser.objects.get(user=user_obj)
print(allocationuser_obj.status)
allocationuser_obj.status= AllocationUserStatusChoice.objects.get(name='Active')
except AllocationUser.DoesNotExist:
# create allocationuser object
allocationuser_obj = AllocationUser(
allocation=allocation,
user=user_obj,
status=AllocationUserStatusChoice.objects.get(name='Active'),
)

# only updating allocation user object
usage_string = user_json_dict[json_user]['usage']
num, alpha = splitString(usage_string)
allocationuser_obj.usage = num
allocationuser_obj.usage_bytes = user_json_dict[json_user]['logical_usage']
allocationuser_obj.unit = alpha
allocationuser_obj.allocation_group_usage_bytes = lab_data["kbytes"]
allocationuser_obj.allocation_group_quota = lab_data["quota"]
allocationuser_obj.save()
allocation.save()
# get_user_model().objects.get(username=json_user).save()
allocationuser_obj, created = AllocationUser.objects.get_or_create(
user=user_obj,
allocation=allocation,
defaults={
'status':AllocationUserStatusChoice.objects.get(name='Active'),
'usage': num,
'usage_bytes': user_json_dict[json_user]['logical_usage'],
'unit': alpha,
}
)

except Exception as e:
# logger.exception(e)
print(f'Error: {e}')
Loading

0 comments on commit b9113ab

Please sign in to comment.