Skip to content

Commit

Permalink
issue #1171 - progress indicator
Browse files Browse the repository at this point in the history
  • Loading branch information
davmlaw committed Sep 19, 2024
1 parent dca7df2 commit e264cb4
Show file tree
Hide file tree
Showing 6 changed files with 64 additions and 13 deletions.
24 changes: 20 additions & 4 deletions analysis/tasks/analysis_grid_export_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,16 @@ def get_grid_downloadable_file_params_hash(pk, export_type):
return sha256sum_str(f"{pk}-{export_type}")


def _write_cached_generated_file(cgf: CachedGeneratedFile, filename, file_iterator):
def _write_cached_generated_file(cgf: CachedGeneratedFile, total_records, filename, file_iterator):
update_size = max(1000, total_records / 100) # 1% or every 1k records
update_progress_iterator = update_cgf_progress_iterator(file_iterator(), cgf.pk, total_records, update_size)

logging.info("Starting to write %s", filename)
media_root_filename = os.path.join(settings.MEDIA_ROOT, str(uuid.uuid4()), filename)
try:
mk_path_for_file(media_root_filename)
with open(media_root_filename, "w") as f:
for line in file_iterator():
for line in update_progress_iterator:
f.write(line) # Already has newline
cgf.filename = media_root_filename
cgf.task_status = "SUCCESS"
Expand All @@ -58,6 +61,19 @@ def _write_cached_generated_file(cgf: CachedGeneratedFile, filename, file_iterat
cgf.save()


def update_cgf_progress_iterator(iterator, cgf_id, total_records, update_size):
update_size = int(update_size) # make sure int so modulus below will hit
cgf_qs = CachedGeneratedFile.objects.filter(id=cgf_id)
cgf_qs.update(progress=0)

for i, record in enumerate(iterator):
if i % update_size == 0:
progress = i / total_records
cgf_qs.update(progress=progress)
yield record
cgf_qs.update(progress=1, task_status='SUCCESS')


@celery.shared_task
def export_cohort_to_downloadable_file(cohort_id, export_type):
# This should have been created in analysis.views.views_grid.cohort_grid_export
Expand All @@ -74,7 +90,7 @@ def export_cohort_to_downloadable_file(cohort_id, export_type):

request = FakeRequest(user=admin_bot())
filename, file_iterator = node_grid_get_export_iterator(request, node, export_type, basename=basename)
_write_cached_generated_file(cgf, filename, file_iterator)
_write_cached_generated_file(cgf, node.count, filename, file_iterator)


@celery.shared_task
Expand All @@ -92,4 +108,4 @@ def export_sample_to_downloadable_file(sample_id, export_type):
str(sample.genome_build)])
request = FakeRequest(user=admin_bot())
filename, file_iterator = node_grid_get_export_iterator(request, node, export_type, basename=basename)
_write_cached_generated_file(cgf, filename, file_iterator)
_write_cached_generated_file(cgf, node.count, filename, file_iterator)
17 changes: 17 additions & 0 deletions snpdb/migrations/0146_cachedgeneratedfile_progress.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Generated by Django 4.2.11 on 2024-09-19 02:16

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("snpdb", "0145_one_off_handle_renamed_table_inherited_index"),
]

operations = [
migrations.AddField(
model_name="cachedgeneratedfile",
name="progress",
field=models.FloatField(null=True),
),
]
2 changes: 2 additions & 0 deletions snpdb/models/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ class CachedGeneratedFile(models.Model):
task_status = models.TextField(null=True)
generate_start = models.DateTimeField(null=True)
generate_end = models.DateTimeField(null=True)
progress = models.FloatField(null=True)

class Meta:
unique_together = ("generator", "params_hash")
Expand Down Expand Up @@ -92,6 +93,7 @@ def get_or_create_and_launch(generator, params_hash, task: signature) -> 'Cached
async_result = task.apply_async()
cgf.task_id = async_result.id
cgf.generate_start = timezone.now()
cgf.progress = 0.0
cgf.save()
else:
async_result = AsyncResult(cgf.task_id)
Expand Down
1 change: 0 additions & 1 deletion snpdb/templates/snpdb/data/sample_graphs_tab.html
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
{% load static %}
<script src="{% static 'js/cached_generated_files.js' %}"></script>
<style>
.generated-graph, .generated-graph img {
width: 640px;
Expand Down
10 changes: 6 additions & 4 deletions snpdb/views/views_json.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
import json
import os

from celery.result import AsyncResult
from django.conf import settings
from django.http.response import JsonResponse
from django.shortcuts import get_object_or_404
from django.views.decorators.http import require_POST

from library.django_utils import require_superuser, get_url_from_media_root_filename
from library.django_utils import require_superuser
from snpdb.models import CachedGeneratedFile, Cohort, Sample, VCF, CustomColumnsCollection, TagColorsCollection
from snpdb.tasks.clingen_tasks import populate_clingen_alleles_from_vcf
from snpdb.tasks.cohort_genotype_tasks import create_cohort_genotype_and_launch_task
Expand All @@ -28,8 +27,11 @@ def job_status(request, job_id):
def cached_generated_file_check(request, cgf_id):
cgf = get_object_or_404(CachedGeneratedFile, pk=cgf_id)
cgf.check_ready()
data = {"status": cgf.task_status,
"cgf_id": cgf.id}
data = {
"status": cgf.task_status,
"cgf_id": cgf.id,
"progress": cgf.progress
}

if cgf.exception:
data["exception"] = str(cgf.exception)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,19 @@ function poll_graph_status(graph_selector, poll_url, delete_url) {
}


function poll_cached_generated_file(poll_url, success_func, failure_func) {
function poll_cached_generated_file(poll_url, success_func, failure_func, update_func) {
$.getJSON(poll_url, function (data) {
if (data.status == "SUCCESS") {
success_func(data);
} else if (data.status == 'FAILURE') {
failure_func(data);
} else {
if (update_func) {
update_func(data.progress);
}

const retry_func = function () {
poll_cached_generated_file(poll_url, success_func, failure_func);
poll_cached_generated_file(poll_url, success_func, failure_func, update_func);
};
window.setTimeout(retry_func, freq);
}
Expand Down Expand Up @@ -103,7 +107,18 @@ class AnnotatedFileDownload {
window.location.href = data.url;
}
}
$(this.selector).html(`<span><i class="fas fa-spinner fa-spin"></i> Preparing download...</span>`);
poll_cached_generated_file(this.pollUrl, downloadFile, this.setError)
let spinner = $(`<span><i class="fas fa-spinner fa-spin"></i> Preparing download... </span>`);
let progressIndicator = $("<span></span>");
spinner.append(progressIndicator);

function updateProgress(progress) {
console.log("updateProgress");
let percent = Math.floor(100 * progress);
progressIndicator.empty();
progressIndicator.append(`${percent}% complete`);
}

$(this.selector).html(spinner);
poll_cached_generated_file(this.pollUrl, downloadFile, this.setError, updateProgress)
}
}

0 comments on commit e264cb4

Please sign in to comment.