issue #1171 - generated file download

analysis/tasks/analysis_grid_export_tasks.py

@@ -29,11 +29,13 @@ def _write_cached_generated_file(cgf: CachedGeneratedFile, filename, file_iterat
             for line in file_iterator():
                 f.write(line)  # Already has newline
         cgf.filename = media_root_filename
+        cgf.task_status = "SUCCESS"
         cgf.generate_end = timezone.now()
         logging.info("Wrote %s", media_root_filename)
     except Exception as e:
         logging.error("Failed to write %s: %s", media_root_filename, e)
         cgf.exception = str(e)
+        cgf.task_status = "FAILURE"
     cgf.save()
 
 

analysis/views/views.py

@@ -609,14 +609,14 @@ def node_graph(request, analysis_id, node_id, graph_type_id, cmap):
     get_node_subclass_or_404(request.user, node_id)  # Permission check
     node_graph_type = NodeGraphType.objects.get(pk=graph_type_id)
     cached_graph = graphcache.async_graph(node_graph_type.graph_class, cmap, node_id)
-    return HttpResponseRedirect(reverse("cached_generated_file_check", kwargs={"cgf_id": cached_graph.id}))
+    return redirect(cached_graph)
 
 
 def column_summary_boxplot(request, analysis_id, node_id, label, variant_column):
     get_node_subclass_or_404(request.user, node_id)  # Permission check
     graph_class_name = full_class_name(ColumnBoxplotGraph)
     cached_graph = graphcache.async_graph(graph_class_name, node_id, label, variant_column)
-    return HttpResponseRedirect(reverse("cached_generated_file_check", kwargs={"cgf_id": cached_graph.id}))
+    return redirect(cached_graph)
 
 
 def cohort_zygosity_filters(request, analysis_id, node_id, cohort_id):

analysis/views/views_grid.py

@@ -6,6 +6,7 @@
 from django.core.cache import cache
 from django.http import JsonResponse
 from django.http.response import StreamingHttpResponse, HttpResponseRedirect
+from django.shortcuts import redirect
 from django.urls import reverse
 from django.utils.decorators import method_decorator
 from django.views.decorators.cache import cache_page
@@ -118,7 +119,7 @@ def cohort_grid_export(request, cohort_id, export_type):
     params_hash = get_grid_downloadable_file_params_hash(cohort_id, export_type)
     task = export_cohort_to_downloadable_file.si(cohort_id, export_type)
     cgf = CachedGeneratedFile.get_or_create_and_launch("export_cohort_to_downloadable_file", params_hash, task)
-    return JsonResponse({"celery_task": cgf.task_id})
+    return redirect(cgf)
 
 
 def sample_grid_export(request, sample_id, export_type):

library/django_utils/__init__.py

@@ -1,5 +1,6 @@
 import datetime
 import operator
+import os
 from functools import reduce
 from functools import wraps, partial
 
@@ -39,6 +40,13 @@ def get_url_from_view_path(view_path):
     return f'{protocol}://{current_site.domain}{view_path}'
 
 
+def get_url_from_media_root_filename(filename):
+    media_root_with_slash = os.path.join(settings.MEDIA_ROOT, "")
+    if not filename.startswith(media_root_with_slash):
+        raise ValueError(f"'{filename}' must start with MEDIA_ROOT: {media_root_with_slash}")
+    return os.path.join(settings.MEDIA_URL, filename[len(media_root_with_slash):])
+
+
 def add_save_message(request, valid, name, created=False):
     action = "created" if created else "saved"
 

pedigree/views.py

@@ -4,7 +4,7 @@
 from django.forms.formsets import formset_factory
 from django.forms.models import ModelChoiceField
 from django.http.response import HttpResponseRedirect
-from django.shortcuts import get_object_or_404, render
+from django.shortcuts import get_object_or_404, render, redirect
 from django.urls.base import reverse
 
 from library.utils import full_class_name
@@ -37,7 +37,7 @@ def pedigree_chart(request, ped_file_id):
     ped_file = PedFile.get_for_user(request.user, ped_file_id)  # Make sure we can access it
     graph_class_name = full_class_name(PedigreeChart)
     cached_graph = graphcache.async_graph(graph_class_name, ped_file.pk)
-    return HttpResponseRedirect(reverse("cached_generated_file_check", kwargs={"cgf_id": cached_graph.id}))
+    return redirect(cached_graph)
 
 
 def ped_files(request):

seqauto/views.py

@@ -8,7 +8,7 @@
 from django.core.exceptions import PermissionDenied
 from django.db.models.aggregates import Count
 from django.http.response import HttpResponseRedirect, JsonResponse, HttpResponse
-from django.shortcuts import render, get_object_or_404
+from django.shortcuts import render, get_object_or_404, redirect
 from django.urls.base import reverse
 from django.utils.decorators import method_decorator
 from django.utils.safestring import mark_safe
@@ -478,7 +478,7 @@ def sequencing_run_qc_graph(request, sequencing_run_id, qc_compare_type):
     _ = QCCompareType(qc_compare_type)  # Check valid
     graph_class_name = full_class_name(SequencingRunQCGraph)
     cached_graph = graphcache.async_graph(graph_class_name, sequencing_run_id, qc_compare_type)
-    return HttpResponseRedirect(reverse("cached_generated_file_check", kwargs={"cgf_id": cached_graph.id}))
+    return redirect(cached_graph)
 
 
 def sequencing_run_qc_json_graph(request, sequencing_run_id, qc_compare_type):
@@ -565,13 +565,13 @@ def get_field(f):
 def index_metrics_qc_graph(request, illumina_qc_id):
     graph_class_name = full_class_name(IndexMetricsQCGraph)
     cached_graph = graphcache.async_graph(graph_class_name, illumina_qc_id)
-    return HttpResponseRedirect(reverse("cached_generated_file_check", kwargs={"cgf_id": cached_graph.id}))
+    return redirect(cached_graph)
 
 
 def qc_exec_summary_graph(request, qc_exec_summary_id, qc_compare_type):
     graph_class_name = full_class_name(QCExecSummaryGraph)
     cached_graph = graphcache.async_graph(graph_class_name, qc_exec_summary_id, qc_compare_type)
-    return HttpResponseRedirect(reverse("cached_generated_file_check", kwargs={"cgf_id": cached_graph.id}))
+    return redirect(cached_graph)
 
 
 def qc_exec_summary_json_graph(request, qc_exec_summary_id, qc_compare_type):

snpdb/graphs/graphcache.py

@@ -1,11 +1,8 @@
 import abc
-import logging
 import os
 
 from celery import signature
-from celery.result import AsyncResult
 from django.conf import settings
-from django.utils import timezone
 
 from library.graphs import graph_base
 from library.utils import import_class

snpdb/models/models.py

@@ -10,6 +10,7 @@
 import re
 from dataclasses import dataclass
 from datetime import datetime
+from fileinput import filename
 from functools import cached_property, total_ordering
 from html import escape
 from re import RegexFlag
@@ -33,6 +34,7 @@
 from more_itertools import first
 
 from classification.enums.classification_enums import ShareLevel
+from library.django_utils import get_url_from_media_root_filename
 from library.django_utils.django_object_managers import ObjectManagerCachingRequest
 from library.enums.log_level import LogLevel
 from library.preview_request import PreviewModelMixin
@@ -69,6 +71,14 @@ def __str__(self):
             description = f"task: {self.task_id} sent: {self.generate_start}"
         return f"{self.generator}({self.params_hash}): {description}"
 
+    def get_absolute_url(self):
+        return reverse("cached_generated_file_check", kwargs={"cgf_id": self.pk})
+
+    def get_media_url(self):
+        if self.filename is None:
+            raise ValueError(f"{self}.filename is None")
+        return get_url_from_media_root_filename(self.filename)
+
     @staticmethod
     def get_or_create_and_launch(generator, params_hash, task: signature) -> 'CachedGeneratedFile':
         cgf, created = CachedGeneratedFile.objects.get_or_create(generator=generator,

snpdb/models/models_somalier.py

@@ -14,6 +14,7 @@
 from django_extensions.db.models import TimeStampedModel
 from model_utils.managers import InheritanceManager
 
+from library.django_utils import get_url_from_media_root_filename
 from library.utils import execute_cmd
 from patients.models_enums import Sex
 from pedigree.ped.export_ped import write_unrelated_ped, write_trio_ped
@@ -72,11 +73,7 @@ def sample_name_to_id(sample_name: str):
     @staticmethod
     def media_url(file_path):
         # Need to use a slash, so that later joins don't have absolute path
-        media_root_with_slash = os.path.join(settings.MEDIA_ROOT, "")
-        if not file_path.startswith(media_root_with_slash):
-            raise ValueError(f"'{file_path}' must start with MEDIA_ROOT: {media_root_with_slash}")
-
-        return os.path.join(settings.MEDIA_URL, file_path[len(media_root_with_slash):])
+        return get_url_from_media_root_filename(file_path)
 
 
 class SomalierVCFExtract(AbstractSomalierModel):

snpdb/templates/snpdb/data/view_vcf.html

@@ -10,6 +10,7 @@
 {% block title %}{{ vcf.name }}{% endblock %}
 {% block head %}
 <script type="text/javascript" src="{% static 'js/lib/plotly-latest.min.js' %}"></script>
+<script type="text/javascript" src="{% static 'js/generated_graphs.js' %}"></script>
 <script type="text/javascript" src="{% static 'js/plotly_helper.js' %}"></script>
 
 <style>
@@ -281,6 +282,43 @@
             },
         });
     });
+
+    function pollAndDownloadFile(selector, fileType) {
+        const spinnerId = 'spinner-' + fileType;
+        function downloadFile(data) {
+            /*
+            $("#" + spinnerId).replaceWith('<i class="fas fa-xmark"></i> Error retrieving download...');
+
+                                <a href="{{ annotated_vcf_url }}">
+                                    <div class="icon24 left margin-r-5 vcf-icon"></div> Annotated VCF
+                                </a>
+
+
+             */
+            window.location.href = data.url;
+        }
+
+        let failureFunc = function(data) {
+            console.log("Fail!");
+            $("#" + spinnerId).replaceWith('<i class="fas fa-xmark"></i> Error retrieving download...');
+        }
+
+        $(selector).replaceWith(`<span id="${spinnerId}"><i class="fas fa-spinner fa-spin"></i> Preparing download...</span>`);
+        let pollUrl = Urls.cohort_grid_export({{ vcf.cohort.pk  }}, fileType);
+        poll_cached_generated_file(pollUrl, downloadFile, failureFunc);
+    }
+
+    $("#generate-vcf-download").click(function(event) {
+        event.preventDefault();
+        pollAndDownloadFile(this, "vcf");
+    });
+
+
+    $("#generate-csv-download").click(function(event) {
+        event.preventDefault();
+        pollAndDownloadFile(this, "csv");
+    });
+
 });
 </script>
 <script type="text/javascript" src="{% static 'js/grid.js' %}"></script>
@@ -321,8 +359,29 @@ <h1>VCF: {{ vcf.name }}</h1>
                         <div class="p-2"><a href="{% url 'download_uploaded_file' vcf.uploadedvcf.uploaded_file.pk %}"><div class="icon24 left margin-r-5 vcf-icon"></div> Original VCF</a> ({{ vcf.uploadedvcf.uploaded_file.size|filesizeformat }})</div>
                     {% endif %}
                     {% if can_download_annotated_vcf %}
-                        <div class="p-2"><a href="{% url 'cohort_grid_export' vcf.cohort.pk 'vcf' %}"><div class="icon24 left margin-r-5 vcf-icon"></div> Annotated VCF</a></div>
-                        <div class="p-2"><a href="{% url 'cohort_grid_export' vcf.cohort.pk 'csv' %}"><div class="icon24 left margin-r-5 csv-icon"></div> Annotated CSV</a></div>
+                        <div class="p-2">
+                            {% if annotated_vcf_url %}
+                                <a href="{{ annotated_vcf_url }}">
+                                    <div class="icon24 left margin-r-5 vcf-icon"></div> Annotated VCF
+                                </a>
+                            {% else %}
+                                <a href="#" id="generate-vcf-download">
+                                    <div class="icon24 left margin-r-5 vcf-icon"></div> Annotated VCF
+                                </a>
+                            {% endif %}
+                        </div>
+                        <div class="p-2">
+                            {% if annotated_csv_url %}
+                                <a href="{{ annotated_csv_url }}">
+                                    <div class="icon24 left margin-r-5 csv-icon"></div> Annotated CSV
+                                </a>
+                            {% else %}
+                                <a href="#" id="generate-csv-download">
+                                    <div class="icon24 left margin-r-5 csv-icon"></div> Annotated CSV
+                                </a>
+                            {% endif %}
+
+                        </div>
                     {% endif %}
                 </div>
             </div>

snpdb/views/views.py

@@ -1,6 +1,7 @@
 import itertools
 import json
 import logging
+import os
 from collections import OrderedDict, defaultdict
 from typing import Iterable
 
@@ -27,6 +28,7 @@
 from analysis.analysis_templates import get_sample_analysis
 from analysis.forms import AnalysisOutputNodeChoiceForm
 from analysis.models import AnalysisTemplate
+from analysis.tasks.analysis_grid_export_tasks import get_grid_downloadable_file_params_hash
 from annotation.forms import GeneCountTypeChoiceForm
 from annotation.manual_variant_entry import create_manual_variants, can_create_variants
 from annotation.models import AnnotationVersion, SampleVariantAnnotationStats, SampleGeneAnnotationStats, \
@@ -245,9 +247,11 @@ def view_vcf(request, vcf_id):
 
         add_save_message(request, valid, "VCF")
 
+    cohort_id = None
     try:
         # Some legacy data was too hard to fix and relies on being re-imported
         _ = vcf.cohort
+        cohort_id = vcf.cohort.pk
         _ = vcf.cohort.cohort_genotype_collection
     except (Cohort.DoesNotExist, CohortGenotypeCollection.DoesNotExist):
         messages.add_message(request, messages.ERROR, "This legacy VCF is missing data and needs to be reloaded.")
@@ -279,10 +283,20 @@ def view_vcf(request, vcf_id):
         can_view_upload_pipeline = False
 
     can_download_annotated_vcf = False
-    if vcf.import_status == ImportStatus.SUCCESS:
+    annotated_vcf_url = None
+    annotated_csv_url = None
+    if vcf.import_status == ImportStatus.SUCCESS and cohort_id:
         try:
             AnalysisTemplate.get_template_from_setting("ANALYSIS_TEMPLATES_AUTO_COHORT_EXPORT")
             can_download_annotated_vcf = True
+            params_hash_vcf = get_grid_downloadable_file_params_hash(cohort_id, "vcf")
+            if cgf_vcf := CachedGeneratedFile.objects.filter(generator="export_cohort_to_downloadable_file",
+                                                             params_hash=params_hash_vcf).first():
+                annotated_vcf_url = cgf_vcf.get_media_url()
+            params_hash_csv = get_grid_downloadable_file_params_hash(cohort_id, "csv")
+            if cgf_csv := CachedGeneratedFile.objects.filter(generator="export_cohort_to_downloadable_file",
+                                                             params_hash=params_hash_csv).first():
+                annotated_csv_url = cgf_csv.get_media_url()
         except ValueError:
             pass
 
@@ -299,6 +313,8 @@ def view_vcf(request, vcf_id):
         'can_download_vcf': (not settings.VCF_DOWNLOAD_ADMIN_ONLY) or request.user.is_superuser,
         'can_download_annotated_vcf': can_download_annotated_vcf,
         'can_view_upload_pipeline': can_view_upload_pipeline,
+        'annotated_vcf_url': annotated_vcf_url,
+        'annotated_csv_url': annotated_csv_url,
         "variant_zygosity_count_collections": variant_zygosity_count_collections,
     }
     return render(request, 'snpdb/data/view_vcf.html', context)
@@ -1619,26 +1635,25 @@ def global_sample_gene_matrix(request):
 def genomic_intervals_graph(request, genomic_intervals_collection_id):
     graph_class_name = full_class_name(ChromosomeIntervalsGraph)
     cached_graph = graphcache.async_graph(graph_class_name, genomic_intervals_collection_id)
-    return HttpResponseRedirect(reverse("cached_generated_file_check", kwargs={"cgf_id": cached_graph.id}))
+    return redirect(cached_graph)
 
 
 def chrom_density_graph(request, sample_id, cmap):
     graph_class_name = full_class_name(SampleChromosomeDensityGraph)
-
     cached_graph = graphcache.async_graph(graph_class_name, cmap, sample_id)
-    return HttpResponseRedirect(reverse("cached_generated_file_check", kwargs={"cgf_id": cached_graph.id}))
+    return redirect(cached_graph)
 
 
 def homozygosity_graph(request, sample_id, cmap):
     graph_class_name = full_class_name(HomozygosityPercentGraph)
     cached_graph = graphcache.async_graph(graph_class_name, cmap, sample_id)
-    return HttpResponseRedirect(reverse("cached_generated_file_check", kwargs={"cgf_id": cached_graph.id}))
+    return redirect(cached_graph)
 
 
 def sample_allele_frequency_histogram_graph(request, sample_id, min_read_depth):
     graph_class_name = full_class_name(AlleleFrequencyHistogramGraph)
     cached_graph = graphcache.async_graph(graph_class_name, sample_id, min_read_depth)
-    return HttpResponseRedirect(reverse("cached_generated_file_check", kwargs={"cgf_id": cached_graph.id}))
+    return redirect(cached_graph)
 
 
 def view_genome_build(request, genome_build_name):

snpdb/views/views_json.py

@@ -7,7 +7,7 @@
 from django.shortcuts import get_object_or_404
 from django.views.decorators.http import require_POST
 
-from library.django_utils import require_superuser
+from library.django_utils import require_superuser, get_url_from_media_root_filename
 from snpdb.models import CachedGeneratedFile, Cohort, Sample, VCF, CustomColumnsCollection, TagColorsCollection
 from snpdb.tasks.clingen_tasks import populate_clingen_alleles_from_vcf
 from snpdb.tasks.cohort_genotype_tasks import create_cohort_genotype_and_launch_task
@@ -34,9 +34,7 @@ def cached_generated_file_check(request, cgf_id):
     if cgf.exception:
         data["exception"] = str(cgf.exception)
     elif cgf.task_status == "SUCCESS":
-        media_root_dir = os.path.join(settings.MEDIA_ROOT, "")  # with end slash
-        file_path = cgf.filename.replace(media_root_dir, "")
-        data["url"] = os.path.join(settings.MEDIA_URL, file_path)
+        data["url"] = cgf.get_media_url()
 
     return JsonResponse(data)
 

variantgrid/static_files/default_static/js/generated_graphs.js

@@ -46,3 +46,18 @@ function poll_graph_status(graph_selector, poll_url, delete_url) {
 	});
 }
 
+
+function poll_cached_generated_file(poll_url, success_func, failure_func) {
+	$.getJSON(poll_url, function(data) {
+		if (data.status == "SUCCESS") {
+			success_func(data);
+		} else if (data.status == 'FAILURE') {
+			failure_func(data);
+		} else {
+			const retry_func = function () {
+				poll_cached_generated_file(poll_url, success_func, failure_func);
+			};
+			window.setTimeout(retry_func, freq);
+		}
+	});
+}