Function name fix, using uuid1mc, time format fix, TaskRegistry fix

.travis.yml

@@ -16,11 +16,11 @@ services:
 before_script:
   - mkdir imgee/static/test_uploads
 script:
-  - nosetests --with-coverage
+  - ./runtests.sh
 install:
   - sudo apt-get -qq install zlib1g-dev libfreetype6-dev liblcms1-dev libwebp-dev libjpeg-dev libpng-dev libfreetype6-dev libtiff4-dev librsvg2-dev ghostscript imagemagick pandoc
   - pip install -r requirements.txt
-  - pip install nose coverage BeautifulSoup4 Pillow
+  - pip install -r test_requirements.txt
 notifications:
   email: false
   slack:

imgee/__init__.py

@@ -20,16 +20,12 @@
 
 from . import models, views
 from .models import db
-from .tasks import TaskRegistry
-
-registry = TaskRegistry(os.getenv('ENV', 'production'))
 
 
 def mkdir_p(dirname):
     if not os.path.exists(dirname):
         os.makedirs(dirname)
 
-
 # Configure the app
 coaster.app.init_app(app)
 migrate = Migrate(app, db)
@@ -48,4 +44,6 @@ def error403(error):
     app.config['MEDIA_DOMAIN'] = app.config['MEDIA_DOMAIN'].split(':', 1)[1]
 mkdir_p(app.config['UPLOADED_FILES_DEST'])
 
-registry.set_connection()
+from .tasks import TaskRegistry
+
+registry = TaskRegistry()

imgee/models/stored_file.py

@@ -58,12 +58,16 @@ def __repr__(self):
     def extn(self):
         return guess_extension(self.mimetype, self.orig_extn) or ''
 
+    @property
+    def filename(self):
+        return self.name + self.extn
+
     def dict_data(self):
         return dict(
             title=self.title,
-            uploaded=self.created_at.strftime('%B %d, %Y'),
+            uploaded=self.created_at.isoformat() + 'Z',
             filesize=app.jinja_env.filters['filesizeformat'](self.size),
-            imgsize='%s x %s' % (self.width, self.height),
+            imgsize=u'%s×%s px' % (self.width, self.height),
             url=url_for('view_image', profile=self.profile.name, image=self.name),
             thumb_url=url_for('get_image', image=self.name, size=app.config.get('THUMBNAIL_SIZE'))
         )

imgee/storage.py

@@ -14,9 +14,9 @@
 from imgee import app
 from imgee.models import db, Thumbnail, StoredFile
 from imgee.utils import (
-    newid, guess_extension, get_file_type,
+    newid, guess_extension, get_file_type, is_animated_gif,
     path_for, get_s3_folder, get_s3_bucket,
-    download_frm_s3, get_width_height, ALLOWED_MIMETYPES,
+    download_from_s3, get_width_height, ALLOWED_MIMETYPES,
     exists_in_s3, THUMBNAIL_COMMANDS
 )
 
@@ -29,17 +29,23 @@ def get_resized_image(img, size, is_thumbnail=False):
     """
     registry = imgee.registry
     img_name = img.name
+
+    if img.mimetype == 'image/gif':
+        # if the gif file is animated, no need to resize
+        src_path = download_from_s3(img.filename)
+        if is_animated_gif(src_path):
+            return img.name
+
     size_t = parse_size(size)
     if (size_t and size_t[0] != img.width and size_t[1] != img.height) or ('thumb_extn' in ALLOWED_MIMETYPES[img.mimetype] and ALLOWED_MIMETYPES[img.mimetype]['thumb_extn'] != img.extn):
         w_or_h = or_(Thumbnail.width == size_t[0], Thumbnail.height == size_t[1])
         scaled = Thumbnail.query.filter(w_or_h, Thumbnail.stored_file == img).first()
         if scaled and exists_in_s3(scaled):
             img_name = scaled.name
         else:
-            size = get_fitting_size((img.width, img.height), size_t)
+            original_size = (img.width, img.height)
+            size = get_fitting_size(original_size, size_t)
             resized_filename = get_resized_filename(img, size)
-            registry = imgee.registry
-
             try:
                 if resized_filename in registry:
                     # file is still being processed
@@ -48,12 +54,10 @@ def get_resized_image(img, size, is_thumbnail=False):
                 else:
                     registry.add(resized_filename)
                 img_name = resize_and_save(img, size, is_thumbnail=is_thumbnail)
-            except Exception as e:
-                # something broke while processing the file
-                raise e
             finally:
                 # file has been processed, remove from registry
-                registry.remove(resized_filename)
+                if resized_filename in registry:
+                    registry.remove(resized_filename)
     return img_name
 
 
@@ -74,7 +78,7 @@ def save_file(fp, profile, title=None):
 
     stored_file = save_img_in_db(name=id_, title=title, local_path=local_path,
                     profile=profile, mimetype=content_type, orig_extn=extn)
-    s3resp = save_on_s3(img_name, content_type=content_type)
+    save_on_s3(img_name, content_type=content_type)
     return title, stored_file
 
 
@@ -121,7 +125,7 @@ def save_on_s3(filename, remotename='', content_type='', bucket='', folder=''):
         headers = {
             'Cache-Control': 'max-age=31536000',  # 60*60*24*365
             'Content-Type': get_file_type(fp, filename),
-            'Expires': datetime.now() + timedelta(days=365)
+            'Expires': str(datetime.utcnow() + timedelta(days=365))
         }
         k.set_contents_from_file(fp, policy='public-read', headers=headers)
     return filename
@@ -146,7 +150,7 @@ def parse_size(size):
         return tuple(map(int, size))
 
 
-def get_fitting_size((orig_w, orig_h), size):
+def get_fitting_size(original_size, size):
     """
      Return the size to fit the image to the box
      along the smaller side and preserve aspect ratio.
@@ -171,6 +175,8 @@ def get_fitting_size((orig_w, orig_h), size):
     >>> get_fitting_size((200, 500), (400, 600))
     [240, 600]
     """
+    orig_w, orig_h = original_size
+
     if orig_w == 0 or orig_h == 0:
         # this is either a cdr file or a zero width file
         # just go with target size
@@ -216,7 +222,7 @@ def resize_and_save(img, size, is_thumbnail=False):
     Get the original image from local disk cache, download it from S3 if it misses.
     Resize the image and save resized image on S3 and size details in db.
     """
-    src_path = download_frm_s3(img.name + img.extn)
+    src_path = download_from_s3(img.filename)
 
     if 'thumb_extn' in ALLOWED_MIMETYPES[img.mimetype]:
         format = ALLOWED_MIMETYPES[img.mimetype]['thumb_extn']
@@ -258,7 +264,8 @@ def resize_img(src, dest, size, mimetype, format, is_thumbnail):
 
 def clean_local_cache(expiry=24):
     """
-    Remove files from local cache which are NOT accessed in the last `expiry` hours.
+    Remove files from local cache
+    which are NOT accessed in the last `expiry` hours.
     """
     cache_path = app.config.get('UPLOADED_FILES_DEST')
     cache_path = os.path.join(cache_path, '*')
@@ -275,8 +282,12 @@ def clean_local_cache(expiry=24):
 def delete(stored_file, commit=True):
     """
     Delete all the thumbnails and images associated with a file, from local cache and S3.
-    Wait for the upload/resize to complete if queued for the same image.
     """
+    registry = imgee.registry
+
+    # remove from registry
+    registry.remove_keys_starting_with(stored_file.name)
+
     # remove locally
     cache_path = app.config.get('UPLOADED_FILES_DEST')
     cached_img_path = os.path.join(cache_path, '%s*' % stored_file.name)

imgee/tasks.py

@@ -1,35 +1,68 @@
+import re
 import redis
 from imgee import app
 
 
 class TaskRegistry(object):
     def __init__(self, name='default', connection=None):
-        self.connection = redis.from_url(connection) if connection else None
+        self.connection = connection or self.set_connection_from_url()
         self.name = name
-        self.key = 'imgee:registry:%s' % name
+        self.key_prefix = 'imgee:registry:%s' % name
+        self.filename_pattern = '^[a-z0-9\_\.]+$'
+        self.pipe = self.connection.pipeline()
 
-    def set_connection(self, connection=None):
-        connection = connection or app.config.get('REDIS_URL')
-        self.connection = redis.from_url(connection)
+    def set_connection_from_url(self, url=None):
+        url = url or app.config.get('REDIS_URL')
+        self.connection = redis.from_url(url)
+        return self.connection
 
-    def add(self, taskid):
-        self.connection.sadd(self.key, taskid)
-
-    def remove(self, taskid):
-        self.connection.srem(self.key, taskid)
-
-    def remove_all(self):
-        for k in self.get_all_keys():
-            self.remove(k)
+    def key_for(self, taskid):
+        return u'{key_prefix}:{taskid}'.format(key_prefix=self.key_prefix, taskid=taskid)
 
     def __contains__(self, taskid):
-        return self.connection.sismember(self.key, taskid)
+        return len(self.connection.keys(self.key_for(taskid))) > 0
 
-    def keys_starting_with(self, query):
-        return filter(lambda k: k.startswith(query), self.connection.smembers(self.key))
+    def add(self, taskid, expire=60):
+        self.connection.set(self.key_for(taskid), taskid)
+        # setting TTL of 60 seconds for the key
+        # if the file doesn't get processed within 60 seconds,
+        # it'll be removed from the registry
+        self.connection.expire(self.key_for(taskid), expire)
 
     def search(self, query):
-        return filter(lambda k: str(query) in k, self.connection.smembers(self.key))
+        # >> KEYS imgee:registry:default:*query*
+        if not re.compile(self.filename_pattern).match(query):
+            return list()
+        return self.connection.keys(self.key_for('*{}*'.format(query)))
 
     def get_all_keys(self):
-        return list(self.connection.smembers(self.key))
+        # >> KEYS imgee:registry:default:*
+        return self.connection.keys(self.key_for('*'))
+
+    def keys_starting_with(self, query):
+        # >> KEYS imgee:registry:default:query*
+        if not re.compile(self.filename_pattern).match(query):
+            return list()
+        return self.connection.keys(self.key_for('{}*'.format(query)))
+
+    def remove(self, taskid):
+        # remove a key with the taskid
+        self.remove_by_key(self.key_for(taskid))
+
+    def remove_by_key(self, key):
+        # remove a redis key directly
+        self.connection.delete(key)
+
+    def remove_keys(self, keys):
+        for k in keys:
+            self.pipe.delete(k)
+        self.pipe.execute()
+
+    def remove_keys_matching(self, query):
+        self.remove_keys(self.search(query))
+
+    def remove_keys_starting_with(self, query):
+        self.remove_keys(self.keys_starting_with(query))
+
+    def remove_all(self):
+        self.remove_keys(self.get_all_keys())

imgee/utils.py

@@ -4,15 +4,16 @@
 import os.path
 from subprocess import check_output, CalledProcessError
 from urlparse import urljoin
-from uuid import uuid4
 
 from boto import connect_s3
 from boto.s3.bucket import Bucket
 from boto.s3.key import Key
 import defusedxml.cElementTree as elementtree
 from flask import request
 import magic
+from PIL import Image
 
+from coaster.utils import uuid1mc
 import imgee
 from imgee import app
 
@@ -84,7 +85,7 @@
 
 
 def newid():
-    return unicode(uuid4().hex)
+    return unicode(uuid1mc().hex)
 
 
 def get_media_domain(scheme=None):
@@ -131,6 +132,16 @@ def is_svg(fp):
     return tag == '{http://www.w3.org/2000/svg}svg'
 
 
+def is_animated_gif(local_path):
+    is_animated = True
+    gif = Image.open(local_path)
+    try:
+        gif.seek(1)
+    except EOFError:
+        is_animated = False
+    return is_animated
+
+
 def get_file_type(fp, filename=None):
     fp.seek(0)
     data = fp.read()
@@ -186,7 +197,7 @@ def exists_in_s3(thumb):
     return True
 
 
-def download_frm_s3(img_name):
+def download_from_s3(img_name):
     local_path = path_for(img_name)
     if not os.path.exists(local_path):
         bucket = get_s3_bucket()

requirements.txt

@@ -13,3 +13,4 @@ xml4h
 python-magic
 defusedxml
 Flask-Migrate
+pillow

runtests.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+export FLASK_ENV="TESTING"
+coverage run `which nosetests --with-timer`
+coverage report

test_requirements.txt

@@ -0,0 +1,2 @@
+nose
+coverage