From e264cb48a6d5e83afddc5726bc7e1cbcfd79c5f7 Mon Sep 17 00:00:00 2001 From: Dave Lawrence Date: Thu, 19 Sep 2024 13:10:59 +0930 Subject: [PATCH] issue #1171 - progress indicator --- analysis/tasks/analysis_grid_export_tasks.py | 24 +++++++++++++++---- .../0146_cachedgeneratedfile_progress.py | 17 +++++++++++++ snpdb/models/models.py | 2 ++ .../snpdb/data/sample_graphs_tab.html | 1 - snpdb/views/views_json.py | 10 ++++---- .../js/cached_generated_files.js | 23 ++++++++++++++---- 6 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 snpdb/migrations/0146_cachedgeneratedfile_progress.py diff --git a/analysis/tasks/analysis_grid_export_tasks.py b/analysis/tasks/analysis_grid_export_tasks.py index 910eab253..4d31cdb62 100644 --- a/analysis/tasks/analysis_grid_export_tasks.py +++ b/analysis/tasks/analysis_grid_export_tasks.py @@ -39,13 +39,16 @@ def get_grid_downloadable_file_params_hash(pk, export_type): return sha256sum_str(f"{pk}-{export_type}") -def _write_cached_generated_file(cgf: CachedGeneratedFile, filename, file_iterator): +def _write_cached_generated_file(cgf: CachedGeneratedFile, total_records, filename, file_iterator): + update_size = max(1000, total_records / 100) # 1% or every 1k records + update_progress_iterator = update_cgf_progress_iterator(file_iterator(), cgf.pk, total_records, update_size) + logging.info("Starting to write %s", filename) media_root_filename = os.path.join(settings.MEDIA_ROOT, str(uuid.uuid4()), filename) try: mk_path_for_file(media_root_filename) with open(media_root_filename, "w") as f: - for line in file_iterator(): + for line in update_progress_iterator: f.write(line) # Already has newline cgf.filename = media_root_filename cgf.task_status = "SUCCESS" @@ -58,6 +61,19 @@ def _write_cached_generated_file(cgf: CachedGeneratedFile, filename, file_iterat cgf.save() +def update_cgf_progress_iterator(iterator, cgf_id, total_records, update_size): + update_size = int(update_size) # make sure int so modulus below will hit + cgf_qs = CachedGeneratedFile.objects.filter(id=cgf_id) + cgf_qs.update(progress=0) + + for i, record in enumerate(iterator): + if i % update_size == 0: + progress = i / total_records + cgf_qs.update(progress=progress) + yield record + cgf_qs.update(progress=1, task_status='SUCCESS') + + @celery.shared_task def export_cohort_to_downloadable_file(cohort_id, export_type): # This should have been created in analysis.views.views_grid.cohort_grid_export @@ -74,7 +90,7 @@ def export_cohort_to_downloadable_file(cohort_id, export_type): request = FakeRequest(user=admin_bot()) filename, file_iterator = node_grid_get_export_iterator(request, node, export_type, basename=basename) - _write_cached_generated_file(cgf, filename, file_iterator) + _write_cached_generated_file(cgf, node.count, filename, file_iterator) @celery.shared_task @@ -92,4 +108,4 @@ def export_sample_to_downloadable_file(sample_id, export_type): str(sample.genome_build)]) request = FakeRequest(user=admin_bot()) filename, file_iterator = node_grid_get_export_iterator(request, node, export_type, basename=basename) - _write_cached_generated_file(cgf, filename, file_iterator) + _write_cached_generated_file(cgf, node.count, filename, file_iterator) diff --git a/snpdb/migrations/0146_cachedgeneratedfile_progress.py b/snpdb/migrations/0146_cachedgeneratedfile_progress.py new file mode 100644 index 000000000..bbe7d015c --- /dev/null +++ b/snpdb/migrations/0146_cachedgeneratedfile_progress.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.11 on 2024-09-19 02:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("snpdb", "0145_one_off_handle_renamed_table_inherited_index"), + ] + + operations = [ + migrations.AddField( + model_name="cachedgeneratedfile", + name="progress", + field=models.FloatField(null=True), + ), + ] diff --git a/snpdb/models/models.py b/snpdb/models/models.py index 7a4079c7f..5030727e2 100644 --- a/snpdb/models/models.py +++ b/snpdb/models/models.py @@ -63,6 +63,7 @@ class CachedGeneratedFile(models.Model): task_status = models.TextField(null=True) generate_start = models.DateTimeField(null=True) generate_end = models.DateTimeField(null=True) + progress = models.FloatField(null=True) class Meta: unique_together = ("generator", "params_hash") @@ -92,6 +93,7 @@ def get_or_create_and_launch(generator, params_hash, task: signature) -> 'Cached async_result = task.apply_async() cgf.task_id = async_result.id cgf.generate_start = timezone.now() + cgf.progress = 0.0 cgf.save() else: async_result = AsyncResult(cgf.task_id) diff --git a/snpdb/templates/snpdb/data/sample_graphs_tab.html b/snpdb/templates/snpdb/data/sample_graphs_tab.html index ba22ae7f1..70d106893 100644 --- a/snpdb/templates/snpdb/data/sample_graphs_tab.html +++ b/snpdb/templates/snpdb/data/sample_graphs_tab.html @@ -1,5 +1,4 @@ {% load static %} -