Merge pull request #54 from diging/develop

Develop

cookies/giles.py

@@ -27,7 +27,7 @@ def wrapper(user, *args, **kwargs):
                 user.giles_token.delete()
             except AssertionError:
                 pass
-                
+
             get_user_auth_token(user, **kwargs)
             user.refresh_from_db()
             # TODO: we could put some Exception handling here.
@@ -85,14 +85,13 @@ def get_user_auth_token(user, **kwargs):
         user.save()
         return user.giles_token.token
     except Exception as E:
-        template = "Failed to retrieve access token for user {u}: {m}"
-        msg = template.format(u=user.username, m=data.get('errorMsg', 'nope'))
+        template = "Failed to retrieve access token for user {u}"
+        msg = template.format(u=user.username)
         if kwargs.get('raise_exception', False):
             raise E
         logger.error(msg)
 
 
-
 # @handle_status_exception
 @api_request
 def get_auth_token(user, **kwargs):
@@ -472,9 +471,6 @@ def process_resources(user, session, **kwargs):
         _process_document_data(session, file_data, user, **kwargs)
 
 
-
-
-
 def handle_giles_callback(request, **kwargs):
     """
     When a user has uploaded images to Giles, they may visit a callback URL that

cookies/metadata.py

@@ -236,8 +236,8 @@ def filter_relations(source=None, predicate=None, target=None,
     -------
     :class:`django.db.models.query.QuerySet`
     """
-    # if user and not user.is_superuser:
-    #     qs = authorization.apply_filter(user, 'view_relation', qs)
+    if user and not user.is_superuser:
+        qs = authorization.apply_filter(user, 'view_relation', qs)
 
     for field, qfield, value in [('source', 'source_instance_id', source),
                                  ('target', 'target_instance_id', target)]:

cookies/migrations/0027_auto_20161107_1625.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.6 on 2016-11-07 16:25
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cookies', '0026_auto_20161104_1416'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='collection',
+            name='public',
+            field=models.BooleanField(default=False, help_text=b'If a resource is not public it will only be accessible to logged-in users and will not appear in public search results. If this option is selected, you affirm that you have the right to upload and distribute this resource.'),
+        ),
+        migrations.AlterField(
+            model_name='conceptentity',
+            name='public',
+            field=models.BooleanField(default=False, help_text=b'If a resource is not public it will only be accessible to logged-in users and will not appear in public search results. If this option is selected, you affirm that you have the right to upload and distribute this resource.'),
+        ),
+        migrations.AlterField(
+            model_name='relation',
+            name='public',
+            field=models.BooleanField(default=False, help_text=b'If a resource is not public it will only be accessible to logged-in users and will not appear in public search results. If this option is selected, you affirm that you have the right to upload and distribute this resource.'),
+        ),
+        migrations.AlterField(
+            model_name='resource',
+            name='public',
+            field=models.BooleanField(default=False, help_text=b'If a resource is not public it will only be accessible to logged-in users and will not appear in public search results. If this option is selected, you affirm that you have the right to upload and distribute this resource.'),
+        ),
+    ]

cookies/migrations/0028_auto_20161107_1636.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.6 on 2016-11-07 16:36
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cookies', '0027_auto_20161107_1625'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='gilesupload',
+            name='created_by',
+        ),
+        migrations.RemoveField(
+            model_name='gilesupload',
+            name='updated',
+        ),
+        migrations.AddField(
+            model_name='gilesupload',
+            name='content_resource',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='giles_upload', to='cookies.Resource'),
+        ),
+        migrations.AddField(
+            model_name='gilesupload',
+            name='resolved',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='gilesupload',
+            name='response',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='gilesupload',
+            name='sent',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='gilesupload',
+            name='upload_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

cookies/migrations/0029_gilesupload_fail.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.6 on 2016-11-07 17:48
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cookies', '0028_auto_20161107_1636'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='gilesupload',
+            name='fail',
+            field=models.BooleanField(default=False),
+        ),
+    ]

cookies/models.py

@@ -429,6 +429,36 @@ def get_absolute_url(self):
         return reverse('jobs')
 
 
+class GilesUpload(models.Model):
+    """
+    Represents a single upload.
+    """
+
+    created = models.DateTimeField(auto_now_add=True)
+    sent = models.DateTimeField(null=True, blank=True)
+    """The datetime when the file was uploaded."""
+
+    upload_id = models.CharField(max_length=255, blank=True, null=True)
+    """Returned by Giles upon upload."""
+
+    content_resource = models.ForeignKey('Resource', null=True, blank=True,
+                                         related_name='giles_upload')
+    """This is the resource that directly 'owns' the uploaded file."""
+
+    response = models.TextField(blank=True, null=True)
+    """This should be raw JSON."""
+
+    resolved = models.BooleanField(default=False)
+    """When a successful response is received, this should be set ``True``."""
+
+    fail = models.BooleanField(default=False)
+    """If ``True``, should not be retried."""
+
+    @property
+    def pending(self):
+        return not self.resolved
+
+
 class GilesSession(models.Model):
     created_by = models.ForeignKey(User, related_name='giles_sessions')
     created = models.DateTimeField(auto_now_add=True)
@@ -457,14 +487,6 @@ def _set_file_details(self, value):
     file_details = property(_get_file_details, _set_file_details)
 
 
-class GilesUpload(models.Model):
-    """
-    Tracks files that have been uploaded via the REST API.
-    """
-    created_by = models.ForeignKey(User, related_name='giles_uploads')
-    created = models.DateTimeField(auto_now_add=True)
-    updated = models.DateTimeField(auto_now=True)
-
 
 class GilesToken(models.Model):
     """

cookies/signals.py

@@ -39,11 +39,14 @@ def new_users_are_inactive_by_default(sender, **kwargs):
         # instance.save()
 
 
-# TODO: enable this when Giles is ready for asynchronous uploads.
-# @receiver(post_save, sender=ContentRelation)
+@receiver(post_save, sender=ContentRelation)
 def send_pdfs_and_images_to_giles(sender, **kwargs):
+    """
+    Create a :class:`.GilesUpload` instance to indicate that an upload should
+    be performed.
+    """
     instance = kwargs.get('instance', None)
-    print 'received post_Save for ContentRelation %i' % instance.id
+    logger.debug('received post_save for ContentRelation %i' % instance.id)
 
     import mimetypes
     try:
@@ -53,15 +56,8 @@ def send_pdfs_and_images_to_giles(sender, **kwargs):
     if instance.content_resource.is_local and instance.content_resource.file.name is not None:
         # PDFs and images should be stored in Digilib via Giles.
         if content_type in ['image/png', 'image/tiff', 'image/jpeg', 'application/pdf']:
-            logger.debug('%s has a ContentResource; sending to Giles' % instance.content_resource.name)
-            try:
-                task = send_to_giles.delay(instance.content_resource.file.name,
-                                    instance.for_resource.created_by, resource=instance.for_resource,
-                                    public=instance.for_resource.public)
-            except ConnectionError:
-                logger.error("send_pdfs_and_images_to_giles: there was an error"
-                             " connecting to the redis message passing"
-                             " backend.")
+            logger.debug('%s has a ContentResource; creating a GilesUpload' % instance.content_resource.name)
+            GilesUpload.objects.create(content_resource=instance.content_resource)
 
 
 

cookies/tasks.py

@@ -11,7 +11,7 @@
 from cookies.exceptions import *
 logger = settings.LOGGER
 
-import jsonpickle
+import jsonpickle, json, datetime
 
 
 @shared_task
@@ -90,12 +90,24 @@ def handle_bulk(self, file_path, form_data, file_name, job=None,
 
 
 @shared_task
-def send_to_giles(file_name, creator, resource=None, public=True):
-    result = giles.send_to_giles(creator, file_name, resource=resource,
-                                 public=public)
+def send_to_giles(file_name, creator, resource=None, public=True, gilesupload_id=None):
+    upload = GilesUpload.objects.get(pk=gilesupload_id)
+
+    # try:
+    status_code, response_data = giles.send_to_giles(creator, file_name,
+                                                     resource=resource,
+                                                     public=public)
+    # except:
+        # logger.error("send_to_giles: failing permanently for %i" % upload.id)
+        # upload.fail = True
+        # upload.save()
+        # return
+
     session = GilesSession.objects.create(created_by_id=creator.id)
 
-    stat_sucode, response_data = result
+    upload.upload_id = response_data['id']
+    upload.sent = datetime.datetime.now()
+    upload.save()
 
     try:
         check_giles_upload.delay(resource, creator, response_data['id'],
@@ -105,14 +117,23 @@ def send_to_giles(file_name, creator, resource=None, public=True):
                      " the redis message passing backend.")
 
 
+
 @shared_task(max_retries=None, default_retry_delay=10)
-def check_giles_upload(resource, creator, upload_id, checkURL, session_id):
+def check_giles_upload(resource, creator, upload_id, checkURL, session_id,
+                       gilesupload_id):
     status, content = giles.check_upload_status(creator, checkURL)
     if status == 202:    # Accepted.
         logger.debug('Accepted, retrying in 30 seconds')
         raise check_giles_upload.retry()
+
     giles.process_file_upload(resource, creator, content, session_id)
 
+    upload = GilesUpload.objects.get(pk=gilesupload_id)
+    upload.response = json.dumps(content)
+    upload.resolved = True
+    upload.save()
+
+
 
 @shared_task
 def update_authorizations(auths, user, obj, by_user=None, propagate=True):
@@ -123,3 +144,43 @@ def update_authorizations(auths, user, obj, by_user=None, propagate=True):
 @shared_task
 def search_for_concept(lemma):
     authorities.searchall(lemma)
+
+
+@shared_task
+def send_giles_uploads():
+    """
+    Check for outstanding :class:`.GilesUpload`\s, and send as able.
+    """
+    logger.debug('Checking for outstanding GilesUploads')
+    query = Q(resolved=False) & ~Q(sent=None) & Q(fail=False)
+    outstanding = GilesUpload.objects.filter(query)
+    pending = GilesUpload.objects.filter(resolved=False, sent=None, fail=False)
+
+    # We limit the number of simultaneous requests to Giles.
+    if outstanding.count() >= settings.MAX_GILES_UPLOADS or pending.count() == 0:
+        return
+
+    logger.debug('Found GilesUpload, processing...')
+
+    upload = pending.first()
+    content_resource = upload.content_resource
+    creator = content_resource.created_by
+    resource = content_resource.parent.first().for_resource
+
+    anonymous, _ = User.objects.get_or_create(username='AnonymousUser')
+    public = authorization.check_authorization('view', anonymous, content_resource)
+    result = send_to_giles(content_resource.file.name, creator,
+                           resource=resource, public=public,
+                           gilesupload_id=upload.id)
+
+
+# session = GilesSession.objects.create(created_by_id=creator.id)
+#
+# stat_sucode, response_data = result
+#
+# try:
+#     check_giles_upload.delay(resource, creator, response_data['id'],
+#                              response_data['checkUrl'], session.id)
+# except ConnectionError:
+#     logger.error("send_to_giles: there was an error connecting to"
+#                  " the redis message passing backend.")