From 152c153f11865491d8e86ebc2e20acbd087fc440 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 2 May 2017 10:12:23 +0530 Subject: [PATCH 01/39] function name fix, using uuid1mc, time format fix --- .travis.yml | 1 - imgee/__init__.py | 2 +- imgee/models/stored_file.py | 4 ++-- imgee/storage.py | 13 +++++-------- imgee/utils.py | 6 +++--- 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index d724fdf4..e1259dee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,6 @@ script: install: - sudo apt-get -qq install zlib1g-dev libfreetype6-dev liblcms1-dev libwebp-dev libjpeg-dev libpng-dev libfreetype6-dev libtiff4-dev librsvg2-dev ghostscript imagemagick pandoc - pip install -r requirements.txt - - pip install nose coverage BeautifulSoup4 Pillow notifications: email: false slack: diff --git a/imgee/__init__.py b/imgee/__init__.py index 6dc577b6..cfddf932 100644 --- a/imgee/__init__.py +++ b/imgee/__init__.py @@ -21,7 +21,7 @@ from .models import db from .tasks import TaskRegistry -registry = TaskRegistry(os.getenv('ENV', 'production')) +registry = TaskRegistry() def mkdir_p(dirname): diff --git a/imgee/models/stored_file.py b/imgee/models/stored_file.py index 892bf71f..401505e4 100644 --- a/imgee/models/stored_file.py +++ b/imgee/models/stored_file.py @@ -61,9 +61,9 @@ def extn(self): def dict_data(self): return dict( title=self.title, - uploaded=self.created_at.strftime('%B %d, %Y'), + uploaded=self.created_at.isoformat() + 'Z', filesize=app.jinja_env.filters['filesizeformat'](self.size), - imgsize='%s x %s' % (self.width, self.height), + imgsize=u'%s×%s px' % (self.width, self.height), url=url_for('view_image', profile=self.profile.name, image=self.name), thumb_url=url_for('get_image', image=self.name, size=app.config.get('THUMBNAIL_SIZE')) ) diff --git a/imgee/storage.py b/imgee/storage.py index 3ed8244b..1410c9bd 100644 --- a/imgee/storage.py +++ b/imgee/storage.py @@ -16,7 +16,7 @@ from imgee.utils import ( newid, guess_extension, get_file_type, path_for, get_s3_folder, get_s3_bucket, - download_frm_s3, get_width_height, ALLOWED_MIMETYPES, + download_from_s3, get_width_height, ALLOWED_MIMETYPES, exists_in_s3, THUMBNAIL_COMMANDS ) @@ -38,8 +38,6 @@ def get_resized_image(img, size, is_thumbnail=False): else: size = get_fitting_size((img.width, img.height), size_t) resized_filename = get_resized_filename(img, size) - registry = imgee.registry - try: if resized_filename in registry: # file is still being processed @@ -48,9 +46,6 @@ def get_resized_image(img, size, is_thumbnail=False): else: registry.add(resized_filename) img_name = resize_and_save(img, size, is_thumbnail=is_thumbnail) - except Exception as e: - # something broke while processing the file - raise e finally: # file has been processed, remove from registry registry.remove(resized_filename) @@ -121,7 +116,7 @@ def save_on_s3(filename, remotename='', content_type='', bucket='', folder=''): headers = { 'Cache-Control': 'max-age=31536000', # 60*60*24*365 'Content-Type': get_file_type(fp, filename), - 'Expires': datetime.now() + timedelta(days=365) + 'Expires': str(datetime.utcnow() + timedelta(days=365)) } k.set_contents_from_file(fp, policy='public-read', headers=headers) return filename @@ -216,7 +211,7 @@ def resize_and_save(img, size, is_thumbnail=False): Get the original image from local disk cache, download it from S3 if it misses. Resize the image and save resized image on S3 and size details in db. """ - src_path = download_frm_s3(img.name + img.extn) + src_path = download_from_s3(img.name + img.extn) if 'thumb_extn' in ALLOWED_MIMETYPES[img.mimetype]: format = ALLOWED_MIMETYPES[img.mimetype]['thumb_extn'] @@ -277,6 +272,8 @@ def delete(stored_file, commit=True): Delete all the thumbnails and images associated with a file, from local cache and S3. Wait for the upload/resize to complete if queued for the same image. """ + registry = imgee.registry + # remove locally cache_path = app.config.get('UPLOADED_FILES_DEST') cached_img_path = os.path.join(cache_path, '%s*' % stored_file.name) diff --git a/imgee/utils.py b/imgee/utils.py index ad8593c2..bdb9b4b4 100644 --- a/imgee/utils.py +++ b/imgee/utils.py @@ -4,7 +4,6 @@ import os.path from subprocess import check_output, CalledProcessError from urlparse import urljoin -from uuid import uuid4 from boto import connect_s3 from boto.s3.bucket import Bucket @@ -13,6 +12,7 @@ from flask import request import magic +from coaster.utils import uuid1mc import imgee from imgee import app @@ -84,7 +84,7 @@ def newid(): - return unicode(uuid4().hex) + return unicode(uuid1mc().hex) def get_media_domain(scheme=None): @@ -186,7 +186,7 @@ def exists_in_s3(thumb): return True -def download_frm_s3(img_name): +def download_from_s3(img_name): local_path = path_for(img_name) if not os.path.exists(local_path): bucket = get_s3_bucket() From 039cdbdfc7d0d31674526137ea91060803946a8a Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 2 May 2017 14:54:42 +0530 Subject: [PATCH 02/39] deleting keys from registry when deleting a file --- imgee/storage.py | 19 +++++++++++++------ imgee/tasks.py | 5 +++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/imgee/storage.py b/imgee/storage.py index 1410c9bd..4fcf0cd0 100644 --- a/imgee/storage.py +++ b/imgee/storage.py @@ -36,7 +36,8 @@ def get_resized_image(img, size, is_thumbnail=False): if scaled and exists_in_s3(scaled): img_name = scaled.name else: - size = get_fitting_size((img.width, img.height), size_t) + original_size = (img.width, img.height) + size = get_fitting_size(original_size, size_t) resized_filename = get_resized_filename(img, size) try: if resized_filename in registry: @@ -48,7 +49,8 @@ def get_resized_image(img, size, is_thumbnail=False): img_name = resize_and_save(img, size, is_thumbnail=is_thumbnail) finally: # file has been processed, remove from registry - registry.remove(resized_filename) + if resized_filename in registry: + registry.remove(resized_filename) return img_name @@ -69,7 +71,7 @@ def save_file(fp, profile, title=None): stored_file = save_img_in_db(name=id_, title=title, local_path=local_path, profile=profile, mimetype=content_type, orig_extn=extn) - s3resp = save_on_s3(img_name, content_type=content_type) + save_on_s3(img_name, content_type=content_type) return title, stored_file @@ -141,7 +143,7 @@ def parse_size(size): return tuple(map(int, size)) -def get_fitting_size((orig_w, orig_h), size): +def get_fitting_size(original_size, size): """ Return the size to fit the image to the box along the smaller side and preserve aspect ratio. @@ -166,6 +168,8 @@ def get_fitting_size((orig_w, orig_h), size): >>> get_fitting_size((200, 500), (400, 600)) [240, 600] """ + orig_w, orig_h = original_size + if orig_w == 0 or orig_h == 0: # this is either a cdr file or a zero width file # just go with target size @@ -253,7 +257,8 @@ def resize_img(src, dest, size, mimetype, format, is_thumbnail): def clean_local_cache(expiry=24): """ - Remove files from local cache which are NOT accessed in the last `expiry` hours. + Remove files from local cache + which are NOT accessed in the last `expiry` hours. """ cache_path = app.config.get('UPLOADED_FILES_DEST') cache_path = os.path.join(cache_path, '*') @@ -270,10 +275,12 @@ def clean_local_cache(expiry=24): def delete(stored_file, commit=True): """ Delete all the thumbnails and images associated with a file, from local cache and S3. - Wait for the upload/resize to complete if queued for the same image. """ registry = imgee.registry + # remove from registry + registry.remove_keys_starting_with(stored_file.name) + # remove locally cache_path = app.config.get('UPLOADED_FILES_DEST') cached_img_path = os.path.join(cache_path, '%s*' % stored_file.name) diff --git a/imgee/tasks.py b/imgee/tasks.py index 1c257339..2297dd0c 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -18,6 +18,11 @@ def add(self, taskid): def remove(self, taskid): self.connection.srem(self.key, taskid) + def remove_keys_starting_with(self, query): + keys = self.keys_starting_with(query) + for k in keys: + self.remove(k) + def remove_all(self): for k in self.get_all_keys(): self.remove(k) From 6b8e07f8a2503aa2afa010824065d9e8f8652c47 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 9 May 2017 18:09:02 +0530 Subject: [PATCH 03/39] restructured TaskRegistry to use Redis Keys with TTL set to 60 seconds instead of assigning them to a set --- imgee/__init__.py | 7 +++--- imgee/tasks.py | 55 +++++++++++++++++++++++++++++------------------ 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/imgee/__init__.py b/imgee/__init__.py index d9648116..6ce9c035 100644 --- a/imgee/__init__.py +++ b/imgee/__init__.py @@ -20,9 +20,6 @@ from . import models, views from .models import db -from .tasks import TaskRegistry - -registry = TaskRegistry() def mkdir_p(dirname): @@ -48,4 +45,6 @@ def error403(error): app.config['MEDIA_DOMAIN'] = app.config['MEDIA_DOMAIN'].split(':', 1)[1] mkdir_p(app.config['UPLOADED_FILES_DEST']) -registry.set_connection() +from .tasks import TaskRegistry + +registry = TaskRegistry() diff --git a/imgee/tasks.py b/imgee/tasks.py index 2297dd0c..dc7514e2 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -4,37 +4,50 @@ class TaskRegistry(object): def __init__(self, name='default', connection=None): - self.connection = redis.from_url(connection) if connection else None + self.connection = connection or self.set_connection_from_url() self.name = name - self.key = 'imgee:registry:%s' % name + self.key_prefix = 'imgee:registry:%s' % name - def set_connection(self, connection=None): - connection = connection or app.config.get('REDIS_URL') - self.connection = redis.from_url(connection) + def set_connection_from_url(self, url=None): + url = url or app.config.get('REDIS_URL') + self.connection = redis.from_url(url) + return self.connection + + def _key(self, taskid): + return "{prefix}:{taskid}".format(prefix=self.key_prefix, taskid=taskid) + + def __contains__(self, taskid): + return len(self.connection.keys(self._key(taskid))) > 0 def add(self, taskid): - self.connection.sadd(self.key, taskid) + self.connection.set(self._key(taskid), taskid) + # setting TTL of 60 seconds for the key + # if the file doesn't get processed within 60 seconds, + # it'll be removed from the eregistry + self.connection.expire(self._key(taskid), 60) def remove(self, taskid): - self.connection.srem(self.key, taskid) + self.remove_by_key(self._key(taskid)) - def remove_keys_starting_with(self, query): - keys = self.keys_starting_with(query) - for k in keys: - self.remove(k) + def remove_by_key(self, key): + self.connection.delete(key) - def remove_all(self): - for k in self.get_all_keys(): - self.remove(k) + def search(self, query): + return filter(lambda k: str(query) in k, self.connection.keys("*" + self._key(query + "*"))) - def __contains__(self, taskid): - return self.connection.sismember(self.key, taskid) + def get_all_keys(self): + # >> KEYS imgee:registry:default:* + return list(self.connection.keys(self._key("*"))) def keys_starting_with(self, query): - return filter(lambda k: k.startswith(query), self.connection.smembers(self.key)) + # >> KEYS imgee:registry:default:query* + return self.connection.keys(self._key(query + "*")) - def search(self, query): - return filter(lambda k: str(query) in k, self.connection.smembers(self.key)) + def remove_keys_starting_with(self, query): + keys = self.keys_starting_with(query) + for k in keys: + self.remove_by_key(k) - def get_all_keys(self): - return list(self.connection.smembers(self.key)) + def remove_all(self): + for k in self.get_all_keys(): + self.remove_by_key(k) From 221dcde8c0f5c8d8430d305d1e7e17e9055a5bf3 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 9 May 2017 18:15:49 +0530 Subject: [PATCH 04/39] re-adding nose and coverage to travis file, removed it by mistake --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e1259dee..ff5ad1b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ script: install: - sudo apt-get -qq install zlib1g-dev libfreetype6-dev liblcms1-dev libwebp-dev libjpeg-dev libpng-dev libfreetype6-dev libtiff4-dev librsvg2-dev ghostscript imagemagick pandoc - pip install -r requirements.txt + - pip install nose coverage notifications: email: false slack: From ede59b908e08f03bc97ba3a6211494b0676cf59c Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 9 May 2017 18:26:38 +0530 Subject: [PATCH 05/39] using runtests.py file to run tests in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ff5ad1b8..2f7680c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ services: before_script: - mkdir imgee/static/test_uploads script: - - nosetests --with-coverage + - python runtests.py install: - sudo apt-get -qq install zlib1g-dev libfreetype6-dev liblcms1-dev libwebp-dev libjpeg-dev libpng-dev libfreetype6-dev libtiff4-dev librsvg2-dev ghostscript imagemagick pandoc - pip install -r requirements.txt From dfdf09af6ed5d3faaa5639133e99431b400e54a9 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 11 May 2017 11:47:45 +0530 Subject: [PATCH 06/39] reordered functions and better string formating --- imgee/tasks.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/imgee/tasks.py b/imgee/tasks.py index dc7514e2..d0ba2307 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -13,35 +13,43 @@ def set_connection_from_url(self, url=None): self.connection = redis.from_url(url) return self.connection - def _key(self, taskid): - return "{prefix}:{taskid}".format(prefix=self.key_prefix, taskid=taskid) + def _make_key(self, taskid): + return "{key_prefix}:{taskid}".format(key_prefix=self.key_prefix, taskid=taskid) def __contains__(self, taskid): - return len(self.connection.keys(self._key(taskid))) > 0 + return len(self.connection.keys(self._make_key(taskid))) > 0 def add(self, taskid): - self.connection.set(self._key(taskid), taskid) + self.connection.set(self._make_key(taskid), taskid) # setting TTL of 60 seconds for the key # if the file doesn't get processed within 60 seconds, # it'll be removed from the eregistry - self.connection.expire(self._key(taskid), 60) - - def remove(self, taskid): - self.remove_by_key(self._key(taskid)) - - def remove_by_key(self, key): - self.connection.delete(key) + self.connection.expire(self._make_key(taskid), 60) def search(self, query): - return filter(lambda k: str(query) in k, self.connection.keys("*" + self._key(query + "*"))) + # >> KEYS imgee:registry:default:*query* + return self.connection.keys(self._make_key("*{}*".format(query))) def get_all_keys(self): # >> KEYS imgee:registry:default:* - return list(self.connection.keys(self._key("*"))) + return self.connection.keys(self._make_key("*")) def keys_starting_with(self, query): # >> KEYS imgee:registry:default:query* - return self.connection.keys(self._key(query + "*")) + return self.connection.keys(self._make_key("{}*".format(query))) + + def remove(self, taskid): + # remove a key with the taskid + self.remove_by_key(self._make_key(taskid)) + + def remove_by_key(self, key): + # remove a redis key directly + self.connection.delete(key) + + def remove_keys_matching(self, query): + keys = self.search(query) + for k in keys: + self.remove_by_key(k) def remove_keys_starting_with(self, query): keys = self.keys_starting_with(query) From df77ffcc5b551751cd6eb29038b685e4a8d916d3 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Wed, 31 May 2017 10:01:12 +0530 Subject: [PATCH 07/39] made the redis key expiry time an argument --- imgee/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imgee/tasks.py b/imgee/tasks.py index d0ba2307..da65884a 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -19,12 +19,12 @@ def _make_key(self, taskid): def __contains__(self, taskid): return len(self.connection.keys(self._make_key(taskid))) > 0 - def add(self, taskid): + def add(self, taskid, expire=60): self.connection.set(self._make_key(taskid), taskid) # setting TTL of 60 seconds for the key # if the file doesn't get processed within 60 seconds, # it'll be removed from the eregistry - self.connection.expire(self._make_key(taskid), 60) + self.connection.expire(self._make_key(taskid), expire) def search(self, query): # >> KEYS imgee:registry:default:*query* From 2d7ff92968d1082b15e375f9b032766c942519ee Mon Sep 17 00:00:00 2001 From: Bibhas Date: Wed, 31 May 2017 13:49:29 +0530 Subject: [PATCH 08/39] assigning error handler the right way --- imgee/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/imgee/__init__.py b/imgee/__init__.py index 6ce9c035..0b41af12 100644 --- a/imgee/__init__.py +++ b/imgee/__init__.py @@ -26,19 +26,18 @@ def mkdir_p(dirname): if not os.path.exists(dirname): os.makedirs(dirname) - -def error403(error): - return redirect(url_for('login')) - - # Configure the app coaster.app.init_app(app) migrate = Migrate(app, db) baseframe.init_app(app, requires=['baseframe', 'picturefill', 'imgee']) -app.error_handlers[403] = error403 lastuser.init_app(app) lastuser.init_usermanager(UserManager(db, models.User)) + +@app.errorhandler(403) +def error403(error): + return redirect(url_for('login')) + if app.config.get('MEDIA_DOMAIN') and ( app.config['MEDIA_DOMAIN'].startswith('http:') or app.config['MEDIA_DOMAIN'].startswith('https:')): From 95dce28e38284d6a601da8c291874bd567c39e44 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Wed, 31 May 2017 14:19:42 +0530 Subject: [PATCH 09/39] not resizing if the gif image is animated --- imgee/storage.py | 9 ++++++++- imgee/utils.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/imgee/storage.py b/imgee/storage.py index 4fcf0cd0..a02cd1a7 100644 --- a/imgee/storage.py +++ b/imgee/storage.py @@ -14,7 +14,7 @@ from imgee import app from imgee.models import db, Thumbnail, StoredFile from imgee.utils import ( - newid, guess_extension, get_file_type, + newid, guess_extension, get_file_type, is_animated_gif, path_for, get_s3_folder, get_s3_bucket, download_from_s3, get_width_height, ALLOWED_MIMETYPES, exists_in_s3, THUMBNAIL_COMMANDS @@ -29,6 +29,13 @@ def get_resized_image(img, size, is_thumbnail=False): """ registry = imgee.registry img_name = img.name + + if img.mimetype == 'image/gif': + # if the gif file is animated, no need to resize + src_path = download_from_s3(img.name + img.extn) + if is_animated_gif(src_path): + return img.name + size_t = parse_size(size) if (size_t and size_t[0] != img.width and size_t[1] != img.height) or ('thumb_extn' in ALLOWED_MIMETYPES[img.mimetype] and ALLOWED_MIMETYPES[img.mimetype]['thumb_extn'] != img.extn): w_or_h = or_(Thumbnail.width == size_t[0], Thumbnail.height == size_t[1]) diff --git a/imgee/utils.py b/imgee/utils.py index bdb9b4b4..bf794688 100644 --- a/imgee/utils.py +++ b/imgee/utils.py @@ -11,6 +11,7 @@ import defusedxml.cElementTree as elementtree from flask import request import magic +from PIL import Image from coaster.utils import uuid1mc import imgee @@ -131,6 +132,16 @@ def is_svg(fp): return tag == '{http://www.w3.org/2000/svg}svg' +def is_animated_gif(local_path): + is_animated = True + gif = Image.open(local_path) + try: + gif.seek(1) + except EOFError: + is_animated = False + return is_animated + + def get_file_type(fp, filename=None): fp.seek(0) data = fp.read() From 5cead4e6724a0e6510349d3427b42e0bd0bf38be Mon Sep 17 00:00:00 2001 From: Bibhas Date: Wed, 31 May 2017 16:15:30 +0530 Subject: [PATCH 10/39] added pillow as dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 8be7135d..b48d95e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ xml4h python-magic defusedxml Flask-Migrate +pillow From 7a5bcb93eeed190c3d50ece97f69ad4d37e32b2c Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 6 Jun 2017 15:30:02 +0530 Subject: [PATCH 11/39] using redis pipeline to delete multiple keys --- .travis.yml | 4 ++-- imgee/models/stored_file.py | 4 ++++ imgee/storage.py | 4 ++-- imgee/tasks.py | 37 +++++++++++++++++++++++-------------- runtests.sh | 4 ++++ test_requirements.txt | 2 ++ 6 files changed, 37 insertions(+), 18 deletions(-) create mode 100755 runtests.sh create mode 100644 test_requirements.txt diff --git a/.travis.yml b/.travis.yml index 2f7680c2..4a17fc82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,11 +16,11 @@ services: before_script: - mkdir imgee/static/test_uploads script: - - python runtests.py + - ./runtests.sh install: - sudo apt-get -qq install zlib1g-dev libfreetype6-dev liblcms1-dev libwebp-dev libjpeg-dev libpng-dev libfreetype6-dev libtiff4-dev librsvg2-dev ghostscript imagemagick pandoc - pip install -r requirements.txt - - pip install nose coverage + - pip install -r test_requirements.txt notifications: email: false slack: diff --git a/imgee/models/stored_file.py b/imgee/models/stored_file.py index 401505e4..f58cd6ff 100644 --- a/imgee/models/stored_file.py +++ b/imgee/models/stored_file.py @@ -58,6 +58,10 @@ def __repr__(self): def extn(self): return guess_extension(self.mimetype, self.orig_extn) or '' + @property + def filename(self): + return self.name + self.extn + def dict_data(self): return dict( title=self.title, diff --git a/imgee/storage.py b/imgee/storage.py index a02cd1a7..466fa29c 100644 --- a/imgee/storage.py +++ b/imgee/storage.py @@ -32,7 +32,7 @@ def get_resized_image(img, size, is_thumbnail=False): if img.mimetype == 'image/gif': # if the gif file is animated, no need to resize - src_path = download_from_s3(img.name + img.extn) + src_path = download_from_s3(img.filename) if is_animated_gif(src_path): return img.name @@ -222,7 +222,7 @@ def resize_and_save(img, size, is_thumbnail=False): Get the original image from local disk cache, download it from S3 if it misses. Resize the image and save resized image on S3 and size details in db. """ - src_path = download_from_s3(img.name + img.extn) + src_path = download_from_s3(img.filename) if 'thumb_extn' in ALLOWED_MIMETYPES[img.mimetype]: format = ALLOWED_MIMETYPES[img.mimetype]['thumb_extn'] diff --git a/imgee/tasks.py b/imgee/tasks.py index da65884a..54b01656 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -1,3 +1,4 @@ +import re import redis from imgee import app @@ -7,55 +8,63 @@ def __init__(self, name='default', connection=None): self.connection = connection or self.set_connection_from_url() self.name = name self.key_prefix = 'imgee:registry:%s' % name + self.filename_pattern = '^[a-z0-9\_\.]+$' + self.pipe = self.connection.pipeline() def set_connection_from_url(self, url=None): url = url or app.config.get('REDIS_URL') self.connection = redis.from_url(url) return self.connection - def _make_key(self, taskid): - return "{key_prefix}:{taskid}".format(key_prefix=self.key_prefix, taskid=taskid) + def key_for(self, taskid): + return u'{key_prefix}:{taskid}'.format(key_prefix=self.key_prefix, taskid=taskid) def __contains__(self, taskid): - return len(self.connection.keys(self._make_key(taskid))) > 0 + return len(self.connection.keys(self.key_for(taskid))) > 0 def add(self, taskid, expire=60): - self.connection.set(self._make_key(taskid), taskid) + self.connection.set(self.key_for(taskid), taskid) # setting TTL of 60 seconds for the key # if the file doesn't get processed within 60 seconds, # it'll be removed from the eregistry - self.connection.expire(self._make_key(taskid), expire) + self.connection.expire(self.key_for(taskid), expire) def search(self, query): # >> KEYS imgee:registry:default:*query* - return self.connection.keys(self._make_key("*{}*".format(query))) + if not re.compile(self.filename_pattern).match(query): + return list() + return self.connection.keys(self.key_for('*{}*'.format(query))) def get_all_keys(self): # >> KEYS imgee:registry:default:* - return self.connection.keys(self._make_key("*")) + return self.connection.keys(self.key_for('*')) def keys_starting_with(self, query): # >> KEYS imgee:registry:default:query* - return self.connection.keys(self._make_key("{}*".format(query))) + if not re.compile(self.filename_pattern).match(query): + return list() + return self.connection.keys(self.key_for('{}*'.format(query))) def remove(self, taskid): # remove a key with the taskid - self.remove_by_key(self._make_key(taskid)) + self.remove_by_key(self.key_for(taskid)) def remove_by_key(self, key): # remove a redis key directly self.connection.delete(key) + def remove_keys(self, keys): + for k in keys: + self.pipe.delete(k) + self.pipe.execute() + def remove_keys_matching(self, query): keys = self.search(query) for k in keys: self.remove_by_key(k) def remove_keys_starting_with(self, query): - keys = self.keys_starting_with(query) - for k in keys: - self.remove_by_key(k) + self.remove_keys(self.keys_starting_with(query)) def remove_all(self): - for k in self.get_all_keys(): - self.remove_by_key(k) + self.remove_keys(self.get_all_keys()) diff --git a/runtests.sh b/runtests.sh new file mode 100755 index 00000000..b9fc1819 --- /dev/null +++ b/runtests.sh @@ -0,0 +1,4 @@ +#!/bin/sh +export FLASK_ENV="TESTING" +coverage run `which nosetests --with-timer` +coverage report diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 00000000..a6c88f59 --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,2 @@ +nose +coverage From 28e656f032bfee38d1152885c067919cd26b8d54 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 6 Jun 2017 15:37:33 +0530 Subject: [PATCH 12/39] missed one place to use redis pipeline --- imgee/tasks.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/imgee/tasks.py b/imgee/tasks.py index 54b01656..a40eae53 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -59,9 +59,7 @@ def remove_keys(self, keys): self.pipe.execute() def remove_keys_matching(self, query): - keys = self.search(query) - for k in keys: - self.remove_by_key(k) + self.remove_keys(self.search(query)) def remove_keys_starting_with(self, query): self.remove_keys(self.keys_starting_with(query)) From 83128dde13f3c68cce3558e992e49abda4e047dc Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 6 Jun 2017 15:48:03 +0530 Subject: [PATCH 13/39] deleted runtests.py, not needed anymore --- runtests.py | 7 ------- 1 file changed, 7 deletions(-) delete mode 100755 runtests.py diff --git a/runtests.py b/runtests.py deleted file mode 100755 index b081e4c7..00000000 --- a/runtests.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python -# -*- coding: iso-8859-15 -*- -import os -import nose - -os.environ['FLASK_ENV'] = 'testing' -nose.main() From 144d4acdb5be9ee86f55ad0287e9d9d0454fc2e8 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 6 Jun 2017 15:55:38 +0530 Subject: [PATCH 14/39] fixed typo --- imgee/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imgee/tasks.py b/imgee/tasks.py index a40eae53..06f5eaa2 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -26,7 +26,7 @@ def add(self, taskid, expire=60): self.connection.set(self.key_for(taskid), taskid) # setting TTL of 60 seconds for the key # if the file doesn't get processed within 60 seconds, - # it'll be removed from the eregistry + # it'll be removed from the registry self.connection.expire(self.key_for(taskid), expire) def search(self, query): From dc961d6c1b6d9d1e20d0a996172ce0bf25f204c9 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 8 Jun 2017 14:14:10 +0530 Subject: [PATCH 15/39] fixed some import issues --- imgee/__init__.py | 8 ++++---- imgee/storage.py | 14 +++++++++----- imgee/tasks.py | 25 ++++++++++++++++++------- imgee/utils.py | 4 ++-- 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/imgee/__init__.py b/imgee/__init__.py index 0b41af12..4d41e30d 100644 --- a/imgee/__init__.py +++ b/imgee/__init__.py @@ -20,18 +20,22 @@ from . import models, views from .models import db +from .tasks import TaskRegistry def mkdir_p(dirname): if not os.path.exists(dirname): os.makedirs(dirname) +registry = TaskRegistry() + # Configure the app coaster.app.init_app(app) migrate = Migrate(app, db) baseframe.init_app(app, requires=['baseframe', 'picturefill', 'imgee']) lastuser.init_app(app) lastuser.init_usermanager(UserManager(db, models.User)) +registry.init_app() @app.errorhandler(403) @@ -43,7 +47,3 @@ def error403(error): app.config['MEDIA_DOMAIN'].startswith('https:')): app.config['MEDIA_DOMAIN'] = app.config['MEDIA_DOMAIN'].split(':', 1)[1] mkdir_p(app.config['UPLOADED_FILES_DEST']) - -from .tasks import TaskRegistry - -registry = TaskRegistry() diff --git a/imgee/storage.py b/imgee/storage.py index 466fa29c..66d13676 100644 --- a/imgee/storage.py +++ b/imgee/storage.py @@ -31,7 +31,9 @@ def get_resized_image(img, size, is_thumbnail=False): img_name = img.name if img.mimetype == 'image/gif': - # if the gif file is animated, no need to resize + # if the gif file is animated, not resizing it for now + # but we will need to resize the gif, keeping animation intact + # https://github.com/hasgeek/imgee/issues/55 src_path = download_from_s3(img.filename) if is_animated_gif(src_path): return img.name @@ -125,7 +127,8 @@ def save_on_s3(filename, remotename='', content_type='', bucket='', folder=''): headers = { 'Cache-Control': 'max-age=31536000', # 60*60*24*365 'Content-Type': get_file_type(fp, filename), - 'Expires': str(datetime.utcnow() + timedelta(days=365)) + # once cached, it is set to expire after a year + 'Expires': datetime.utcnow() + timedelta(days=365) } k.set_contents_from_file(fp, policy='public-read', headers=headers) return filename @@ -284,13 +287,14 @@ def delete(stored_file, commit=True): Delete all the thumbnails and images associated with a file, from local cache and S3. """ registry = imgee.registry - - # remove from registry + # remove all the keys related to the given file name + # this is delete all keys matching `imgee:registry:*` registry.remove_keys_starting_with(stored_file.name) # remove locally cache_path = app.config.get('UPLOADED_FILES_DEST') - cached_img_path = os.path.join(cache_path, '%s*' % stored_file.name) + os.remove(os.path.join(cache_path, '%s' % stored_file.filename)) + cached_img_path = os.path.join(cache_path, '%s_*' % stored_file.name) for f in glob(cached_img_path): os.remove(f) diff --git a/imgee/tasks.py b/imgee/tasks.py index 06f5eaa2..bc814726 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -3,14 +3,25 @@ from imgee import app +class InvalidRedisQueryException(Exception): + pass + + class TaskRegistry(object): - def __init__(self, name='default', connection=None): + def __init__(self, name='default', connection=None, app=None): + if app: + self.init_app(name, connection) + + def init_app(self, name='default', connection=None): self.connection = connection or self.set_connection_from_url() self.name = name self.key_prefix = 'imgee:registry:%s' % name self.filename_pattern = '^[a-z0-9\_\.]+$' self.pipe = self.connection.pipeline() + def is_valid_query(self, query): + return bool(re.match(self.filename_pattern, query)) + def set_connection_from_url(self, url=None): url = url or app.config.get('REDIS_URL') self.connection = redis.from_url(url) @@ -31,8 +42,8 @@ def add(self, taskid, expire=60): def search(self, query): # >> KEYS imgee:registry:default:*query* - if not re.compile(self.filename_pattern).match(query): - return list() + if not self.is_valid_query(query): + raise InvalidRedisQueryException(u'Invalid query for searching redis keys: {}'.format(query)) return self.connection.keys(self.key_for('*{}*'.format(query))) def get_all_keys(self): @@ -41,9 +52,9 @@ def get_all_keys(self): def keys_starting_with(self, query): # >> KEYS imgee:registry:default:query* - if not re.compile(self.filename_pattern).match(query): - return list() - return self.connection.keys(self.key_for('{}*'.format(query))) + if not self.is_valid_query(query): + raise InvalidRedisQueryException(u'Invalid query for searching redis keys, starting with: {}'.format(query)) + return self.connection.keys(self.key_for('{}_*'.format(query))) def remove(self, taskid): # remove a key with the taskid @@ -62,7 +73,7 @@ def remove_keys_matching(self, query): self.remove_keys(self.search(query)) def remove_keys_starting_with(self, query): - self.remove_keys(self.keys_starting_with(query)) + self.remove_keys([self.key_for(query)] + self.keys_starting_with(query)) def remove_all(self): self.remove_keys(self.get_all_keys()) diff --git a/imgee/utils.py b/imgee/utils.py index bf794688..01659da5 100644 --- a/imgee/utils.py +++ b/imgee/utils.py @@ -14,7 +14,6 @@ from PIL import Image from coaster.utils import uuid1mc -import imgee from imgee import app THUMBNAIL_COMMANDS = { @@ -249,11 +248,12 @@ def get_no_previews_url(size): def get_image_url(image, size=None): + from imgee import storage extn = image.extn if size and extn in EXTNS: if image.no_previews: return get_no_previews_url(size) - img_name = imgee.storage.get_resized_image(image, size) + img_name = storage.get_resized_image(image, size) if not img_name: return get_no_previews_url(size) else: From 907c7fefe57acdd169a51f1ff7bc2dbdf365bed2 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 8 Jun 2017 14:19:37 +0530 Subject: [PATCH 16/39] moved upload directory creation to manage.py --- imgee/__init__.py | 6 ------ manage.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/imgee/__init__.py b/imgee/__init__.py index 4d41e30d..4fb30f54 100644 --- a/imgee/__init__.py +++ b/imgee/__init__.py @@ -22,11 +22,6 @@ from .models import db from .tasks import TaskRegistry - -def mkdir_p(dirname): - if not os.path.exists(dirname): - os.makedirs(dirname) - registry = TaskRegistry() # Configure the app @@ -46,4 +41,3 @@ def error403(error): app.config['MEDIA_DOMAIN'].startswith('http:') or app.config['MEDIA_DOMAIN'].startswith('https:')): app.config['MEDIA_DOMAIN'] = app.config['MEDIA_DOMAIN'].split(':', 1)[1] -mkdir_p(app.config['UPLOADED_FILES_DEST']) diff --git a/manage.py b/manage.py index 83393267..f5531500 100755 --- a/manage.py +++ b/manage.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import os from coaster.manage import init_manager @@ -6,7 +7,16 @@ from imgee import app +def mkdir_p(dirname): + if not os.path.exists(dirname): + os.makedirs(dirname) + if __name__ == "__main__": db.init_app(app) manager = init_manager(app, db) + + @manager.command + def init(): + mkdir_p(app.config['UPLOADED_FILES_DEST']) + manager.run() From a1372fb78c947645791e8ddf47262dbee1a87244 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 8 Jun 2017 14:22:45 +0530 Subject: [PATCH 17/39] added some comments --- imgee/tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/imgee/tasks.py b/imgee/tasks.py index bc814726..dca89de9 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -51,7 +51,9 @@ def get_all_keys(self): return self.connection.keys(self.key_for('*')) def keys_starting_with(self, query): - # >> KEYS imgee:registry:default:query* + # >> KEYS imgee:registry:default:query_* + # chances are that we'll use this function to find the + # thumbnail keys, which look like "name_wNN_hNN", hence the _ if not self.is_valid_query(query): raise InvalidRedisQueryException(u'Invalid query for searching redis keys, starting with: {}'.format(query)) return self.connection.keys(self.key_for('{}_*'.format(query))) @@ -73,6 +75,8 @@ def remove_keys_matching(self, query): self.remove_keys(self.search(query)) def remove_keys_starting_with(self, query): + # query will most likely be stored_file.name, So + # Delete keys for only `` and `_*` self.remove_keys([self.key_for(query)] + self.keys_starting_with(query)) def remove_all(self): From 127aa4ce753158666f65e89323f1c7a662cb090e Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 8 Jun 2017 14:24:37 +0530 Subject: [PATCH 18/39] moved importing app inside the only function that uses it --- imgee/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imgee/tasks.py b/imgee/tasks.py index dca89de9..70483fdc 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -1,6 +1,5 @@ import re import redis -from imgee import app class InvalidRedisQueryException(Exception): @@ -23,6 +22,7 @@ def is_valid_query(self, query): return bool(re.match(self.filename_pattern, query)) def set_connection_from_url(self, url=None): + from imgee import app url = url or app.config.get('REDIS_URL') self.connection = redis.from_url(url) return self.connection From e3228e4b3fd3c248523215f38608e0f25e4347e0 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 15 Jun 2017 22:17:08 +0530 Subject: [PATCH 19/39] Update runtests.sh --- runtests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtests.sh b/runtests.sh index b9fc1819..bce5ba7b 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -e export FLASK_ENV="TESTING" -coverage run `which nosetests --with-timer` +coverage run `which nosetests` coverage report From 3ca82e97e84a98cc269072806a0585af9c123ad6 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 15 Jun 2017 22:26:59 +0530 Subject: [PATCH 20/39] Update runtests (testing jenkins) --- runtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtests.sh b/runtests.sh index bce5ba7b..964dff40 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,5 +1,5 @@ #!/bin/sh set -e export FLASK_ENV="TESTING" -coverage run `which nosetests` +coverage run `which nosetests` "$@" coverage report From b581138b1ca2e9663c6caa77031bc47fe7fa0ba6 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 15 Jun 2017 22:31:37 +0530 Subject: [PATCH 21/39] testing jenkins --- runtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtests.sh b/runtests.sh index 964dff40..2e9cb98b 100755 --- a/runtests.sh +++ b/runtests.sh @@ -2,4 +2,4 @@ set -e export FLASK_ENV="TESTING" coverage run `which nosetests` "$@" -coverage report +coverage report -m From 395b86258f63d3d1c387dc05e6a4ed70fd9b491f Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 15 Jun 2017 22:33:09 +0530 Subject: [PATCH 22/39] removed pinned versions --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index b48d95e2..6a81bea2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -Flask==0.10.1 +Flask https://github.com/hasgeek/coaster/zipball/master https://github.com/hasgeek/baseframe/zipball/master -SQLAlchemy==1.1.5 -SQLAlchemy-Utils==0.32.12 -Flask-SQLAlchemy==2.1 +SQLAlchemy +SQLAlchemy-Utils +Flask-SQLAlchemy psycopg2 https://github.com/hasgeek/flask-lastuser/zipball/master https://github.com/jace/flask-alembic/archive/master.zip From 85b505caeb06c8c0b3384409e65a20de9618a0a5 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 15 Jun 2017 22:35:18 +0530 Subject: [PATCH 23/39] testing jenkins --- runtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtests.sh b/runtests.sh index 2e9cb98b..476c0dd0 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,5 +1,5 @@ #!/bin/sh set -e export FLASK_ENV="TESTING" -coverage run `which nosetests` "$@" +coverage run `which nosetests` '$@' coverage report -m From 9d17938d02720d92b1b036efd2e34f8f3a8b74fb Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 15 Jun 2017 22:42:48 +0530 Subject: [PATCH 24/39] test jenkins --- runtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtests.sh b/runtests.sh index 476c0dd0..deb5a66b 100755 --- a/runtests.sh +++ b/runtests.sh @@ -2,4 +2,4 @@ set -e export FLASK_ENV="TESTING" coverage run `which nosetests` '$@' -coverage report -m +coverage report From e42e750779b30374328fcb2d5cdd09116478cd44 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 15 Jun 2017 22:53:15 +0530 Subject: [PATCH 25/39] testing jenkins --- runtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtests.sh b/runtests.sh index deb5a66b..476c0dd0 100755 --- a/runtests.sh +++ b/runtests.sh @@ -2,4 +2,4 @@ set -e export FLASK_ENV="TESTING" coverage run `which nosetests` '$@' -coverage report +coverage report -m From fd8737cd47905f50959d9279dad92e3ec728573e Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 15 Jun 2017 22:56:57 +0530 Subject: [PATCH 26/39] testing jenkins --- runtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtests.sh b/runtests.sh index 476c0dd0..2e9cb98b 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,5 +1,5 @@ #!/bin/sh set -e export FLASK_ENV="TESTING" -coverage run `which nosetests` '$@' +coverage run `which nosetests` "$@" coverage report -m From 0cc523eb8742c07e427b3223469f8975f70de654 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Mon, 19 Jun 2017 09:13:11 +0530 Subject: [PATCH 27/39] added manage.py init to runtests --- runtests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/runtests.sh b/runtests.sh index 2e9cb98b..c41fd5a4 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,5 +1,6 @@ #!/bin/sh set -e export FLASK_ENV="TESTING" +./manage.py init # creates the test upload directory coverage run `which nosetests` "$@" coverage report -m From 64da948126e2e85f907c7285916112733d22c554 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Mon, 19 Jun 2017 09:31:35 +0530 Subject: [PATCH 28/39] removed mkdir from travis config --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4a17fc82..a7aba76f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,8 +13,6 @@ python: - 2.7 services: - redis-server -before_script: - - mkdir imgee/static/test_uploads script: - ./runtests.sh install: From eeb5b96be169bad31ac657e5c3c72f44fa602d07 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 20 Jun 2017 09:09:23 +0530 Subject: [PATCH 29/39] trying to fix travis build --- runtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtests.sh b/runtests.sh index c41fd5a4..ba17b07b 100755 --- a/runtests.sh +++ b/runtests.sh @@ -2,5 +2,5 @@ set -e export FLASK_ENV="TESTING" ./manage.py init # creates the test upload directory -coverage run `which nosetests` "$@" +coverage run `which nosetests` --with-timer "$@" coverage report -m From 81ccc8d3beb36b1c10f50216d053e45c7bcb0a06 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 20 Jun 2017 09:13:02 +0530 Subject: [PATCH 30/39] trying to fix travis build --- runtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtests.sh b/runtests.sh index ba17b07b..0120513c 100755 --- a/runtests.sh +++ b/runtests.sh @@ -2,5 +2,5 @@ set -e export FLASK_ENV="TESTING" ./manage.py init # creates the test upload directory -coverage run `which nosetests` --with-timer "$@" +coverage run `which nosetests --with-timer` "$@" coverage report -m From 7a71a7aafe62793a31f5307e100f9cfd6c75ee59 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 20 Jun 2017 09:21:07 +0530 Subject: [PATCH 31/39] trying to debug travis --- manage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/manage.py b/manage.py index f5531500..b73d5039 100755 --- a/manage.py +++ b/manage.py @@ -17,6 +17,9 @@ def mkdir_p(dirname): @manager.command def init(): + print("$"*40) + print(app.config['UPLOADED_FILES_DEST']) + print(os.environ.get('FLASK_ENV')) mkdir_p(app.config['UPLOADED_FILES_DEST']) manager.run() From f2b4b26f7a927ecb1dce938ad639a5449e75cf6c Mon Sep 17 00:00:00 2001 From: Bibhas Date: Tue, 20 Jun 2017 09:37:01 +0530 Subject: [PATCH 32/39] trying to fix travis build --- instance/testing.py | 2 +- manage.py | 3 --- runtests.sh | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/instance/testing.py b/instance/testing.py index 54a6cf84..f73a9f51 100644 --- a/instance/testing.py +++ b/instance/testing.py @@ -1,5 +1,5 @@ from os import environ -UPLOADED_FILES_DEST = 'imgee/static/test_uploads' +UPLOADED_FILES_DEST = 'tests/imgee/static/test_uploads' AWS_FOLDER = 'test/' UNKNOWN_FILE_THUMBNAIL = 'unknown.jpeg' diff --git a/manage.py b/manage.py index b73d5039..f5531500 100755 --- a/manage.py +++ b/manage.py @@ -17,9 +17,6 @@ def mkdir_p(dirname): @manager.command def init(): - print("$"*40) - print(app.config['UPLOADED_FILES_DEST']) - print(os.environ.get('FLASK_ENV')) mkdir_p(app.config['UPLOADED_FILES_DEST']) manager.run() diff --git a/runtests.sh b/runtests.sh index 0120513c..c41fd5a4 100755 --- a/runtests.sh +++ b/runtests.sh @@ -2,5 +2,5 @@ set -e export FLASK_ENV="TESTING" ./manage.py init # creates the test upload directory -coverage run `which nosetests --with-timer` "$@" +coverage run `which nosetests` "$@" coverage report -m From 154b23cc7fe703f75b48441bf8e819816a412fb5 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Wed, 21 Jun 2017 09:57:13 +0530 Subject: [PATCH 33/39] fixed upload path issue --- imgee/__init__.py | 6 ++++++ imgee/storage.py | 3 ++- imgee/utils.py | 4 ++-- instance/testing.py | 2 +- manage.py | 2 +- runtests.sh | 1 + 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/imgee/__init__.py b/imgee/__init__.py index 4fb30f54..a90e4740 100644 --- a/imgee/__init__.py +++ b/imgee/__init__.py @@ -32,6 +32,12 @@ lastuser.init_usermanager(UserManager(db, models.User)) registry.init_app() +# PYTHONPATH is `pwd` when testing and empty otherwise +# using it to determine the project root +# either get it from environment variable, or it's one level up from this init file +app.project_root = os.environ.get('PYTHONPATH', '') or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +print app.project_root + @app.errorhandler(403) def error403(error): diff --git a/imgee/storage.py b/imgee/storage.py index 66d13676..9a4c2aa9 100644 --- a/imgee/storage.py +++ b/imgee/storage.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta from glob import glob import os.path +import os import re from subprocess import check_call, CalledProcessError import time @@ -292,7 +293,7 @@ def delete(stored_file, commit=True): registry.remove_keys_starting_with(stored_file.name) # remove locally - cache_path = app.config.get('UPLOADED_FILES_DEST') + cache_path = os.path.join(app.project_root, app.config.get('UPLOADED_FILES_DEST')) os.remove(os.path.join(cache_path, '%s' % stored_file.filename)) cached_img_path = os.path.join(cache_path, '%s_*' % stored_file.name) for f in glob(cached_img_path): diff --git a/imgee/utils.py b/imgee/utils.py index 01659da5..7bb0a1da 100644 --- a/imgee/utils.py +++ b/imgee/utils.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import re -import os.path +import os from subprocess import check_output, CalledProcessError from urlparse import urljoin @@ -97,7 +97,7 @@ def get_file_url(scheme=None): def path_for(img_name): - return os.path.join(app.config['UPLOADED_FILES_DEST'], img_name) + return os.path.join(app.project_root, app.config['UPLOADED_FILES_DEST'], img_name) # -- mimetypes and content types diff --git a/instance/testing.py b/instance/testing.py index f73a9f51..54a6cf84 100644 --- a/instance/testing.py +++ b/instance/testing.py @@ -1,5 +1,5 @@ from os import environ -UPLOADED_FILES_DEST = 'tests/imgee/static/test_uploads' +UPLOADED_FILES_DEST = 'imgee/static/test_uploads' AWS_FOLDER = 'test/' UNKNOWN_FILE_THUMBNAIL = 'unknown.jpeg' diff --git a/manage.py b/manage.py index f5531500..3c121259 100755 --- a/manage.py +++ b/manage.py @@ -17,6 +17,6 @@ def mkdir_p(dirname): @manager.command def init(): - mkdir_p(app.config['UPLOADED_FILES_DEST']) + mkdir_p(os.path.join(app.project_root, app.config['UPLOADED_FILES_DEST'])) manager.run() diff --git a/runtests.sh b/runtests.sh index c41fd5a4..e105d6c1 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,6 +1,7 @@ #!/bin/sh set -e export FLASK_ENV="TESTING" +export PYTHONPATH=$(pwd) # setting project root as PYTHONPATH ./manage.py init # creates the test upload directory coverage run `which nosetests` "$@" coverage report -m From e7f46fae57543ef3b1c77c17f3b190d60a53ea22 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Wed, 21 Jun 2017 15:51:41 +0530 Subject: [PATCH 34/39] moar fixes to registry, cleanup, moved back to uuid4 --- .travis.yml | 2 ++ imgee/__init__.py | 14 +++----------- imgee/storage.py | 4 ++-- imgee/tasks.py | 26 ++++++++++++++------------ imgee/utils.py | 6 +++--- instance/settings.py | 1 + instance/testing.py | 2 ++ manage.py | 2 +- runtests.sh | 3 +-- tests/fixtures.py | 7 +++++++ 10 files changed, 36 insertions(+), 31 deletions(-) diff --git a/.travis.yml b/.travis.yml index a7aba76f..8d5d8a9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ python: - 2.7 services: - redis-server +before-script: + - ./manage.py init # creates the test upload directory script: - ./runtests.sh install: diff --git a/imgee/__init__.py b/imgee/__init__.py index a90e4740..6e4efc0d 100644 --- a/imgee/__init__.py +++ b/imgee/__init__.py @@ -24,26 +24,18 @@ registry = TaskRegistry() -# Configure the app +# Configure the application coaster.app.init_app(app) migrate = Migrate(app, db) baseframe.init_app(app, requires=['baseframe', 'picturefill', 'imgee']) lastuser.init_app(app) lastuser.init_usermanager(UserManager(db, models.User)) -registry.init_app() - -# PYTHONPATH is `pwd` when testing and empty otherwise -# using it to determine the project root -# either get it from environment variable, or it's one level up from this init file -app.project_root = os.environ.get('PYTHONPATH', '') or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -print app.project_root +registry.init_app(app) @app.errorhandler(403) def error403(error): return redirect(url_for('login')) -if app.config.get('MEDIA_DOMAIN') and ( - app.config['MEDIA_DOMAIN'].startswith('http:') or - app.config['MEDIA_DOMAIN'].startswith('https:')): +if app.config.get('MEDIA_DOMAIN', '').lower().startswith(('http://', 'https://')): app.config['MEDIA_DOMAIN'] = app.config['MEDIA_DOMAIN'].split(':', 1)[1] diff --git a/imgee/storage.py b/imgee/storage.py index 9a4c2aa9..b2252a3a 100644 --- a/imgee/storage.py +++ b/imgee/storage.py @@ -271,7 +271,7 @@ def clean_local_cache(expiry=24): Remove files from local cache which are NOT accessed in the last `expiry` hours. """ - cache_path = app.config.get('UPLOADED_FILES_DEST') + cache_path = os.path.join(app.static_folder, app.config.get('UPLOADED_FILES_DIR')) cache_path = os.path.join(cache_path, '*') min_atime = time.time() - expiry*60*60 @@ -293,7 +293,7 @@ def delete(stored_file, commit=True): registry.remove_keys_starting_with(stored_file.name) # remove locally - cache_path = os.path.join(app.project_root, app.config.get('UPLOADED_FILES_DEST')) + cache_path = os.path.join(app.static_folder, app.config.get('UPLOADED_FILES_DIR')) os.remove(os.path.join(cache_path, '%s' % stored_file.filename)) cached_img_path = os.path.join(cache_path, '%s_*' % stored_file.name) for f in glob(cached_img_path): diff --git a/imgee/tasks.py b/imgee/tasks.py index 70483fdc..610c360f 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -8,25 +8,27 @@ class InvalidRedisQueryException(Exception): class TaskRegistry(object): def __init__(self, name='default', connection=None, app=None): - if app: - self.init_app(name, connection) - - def init_app(self, name='default', connection=None): - self.connection = connection or self.set_connection_from_url() self.name = name + self.connection = connection self.key_prefix = 'imgee:registry:%s' % name self.filename_pattern = '^[a-z0-9\_\.]+$' - self.pipe = self.connection.pipeline() + + if app: + self.init_app(app) + else: + self.app = None + + def init_app(self, app): + self.app = app + + if not self.connection: + url = self.app.config.get('REDIS_URL') + self.connection = redis.from_url(url) + self.pipe = self.connection.pipeline() def is_valid_query(self, query): return bool(re.match(self.filename_pattern, query)) - def set_connection_from_url(self, url=None): - from imgee import app - url = url or app.config.get('REDIS_URL') - self.connection = redis.from_url(url) - return self.connection - def key_for(self, taskid): return u'{key_prefix}:{taskid}'.format(key_prefix=self.key_prefix, taskid=taskid) diff --git a/imgee/utils.py b/imgee/utils.py index 7bb0a1da..18e7f2ab 100644 --- a/imgee/utils.py +++ b/imgee/utils.py @@ -13,7 +13,7 @@ import magic from PIL import Image -from coaster.utils import uuid1mc +from uuid import uuid4 from imgee import app THUMBNAIL_COMMANDS = { @@ -84,7 +84,7 @@ def newid(): - return unicode(uuid1mc().hex) + return unicode(uuid4().hex) def get_media_domain(scheme=None): @@ -97,7 +97,7 @@ def get_file_url(scheme=None): def path_for(img_name): - return os.path.join(app.project_root, app.config['UPLOADED_FILES_DEST'], img_name) + return os.path.join(app.static_folder, app.config['UPLOADED_FILES_DIR'], img_name) # -- mimetypes and content types diff --git a/instance/settings.py b/instance/settings.py index 161fc352..87ceb3f8 100644 --- a/instance/settings.py +++ b/instance/settings.py @@ -34,6 +34,7 @@ LOGFILE = 'error.log' #: File uploads UPLOADED_FILES_DEST = 'imgee/static/uploads' +UPLOADED_FILES_DIR = 'uploads' # this folder should be inside `imgee/static/` #: S3 Configuration AWS_ACCESS_KEY = 'Set aws access key here' AWS_SECRET_KEY = 'Set aws secret key here' diff --git a/instance/testing.py b/instance/testing.py index 54a6cf84..8b5b1eb2 100644 --- a/instance/testing.py +++ b/instance/testing.py @@ -1,5 +1,7 @@ from os import environ UPLOADED_FILES_DEST = 'imgee/static/test_uploads' +UPLOADED_FILES_DIR = 'test_uploads' + AWS_FOLDER = 'test/' UNKNOWN_FILE_THUMBNAIL = 'unknown.jpeg' diff --git a/manage.py b/manage.py index 3c121259..5483a2e8 100755 --- a/manage.py +++ b/manage.py @@ -17,6 +17,6 @@ def mkdir_p(dirname): @manager.command def init(): - mkdir_p(os.path.join(app.project_root, app.config['UPLOADED_FILES_DEST'])) + mkdir_p(os.path.join(app.static_folder, app.config['UPLOADED_FILES_DIR'])) manager.run() diff --git a/runtests.sh b/runtests.sh index e105d6c1..37ced6f7 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,7 +1,6 @@ #!/bin/sh set -e export FLASK_ENV="TESTING" -export PYTHONPATH=$(pwd) # setting project root as PYTHONPATH -./manage.py init # creates the test upload directory +export PYTHONPATH=$(dirname $0) # setting project root as PYTHONPATH coverage run `which nosetests` "$@" coverage report -m diff --git a/tests/fixtures.py b/tests/fixtures.py index e5aad2b6..eb9e1c97 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,3 +1,4 @@ +import os import unittest import random import string @@ -25,6 +26,12 @@ def get_test_profile(self): def setUp(self): app.config['TESTING'] = True app.testing = True + + # PYTHONPATH is `pwd` when testing and empty otherwise + # using it to determine the project root + # either get it from environment variable, or it's one level up from this init file + app.project_root = os.environ.get('PYTHONPATH', '') or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + db.create_all() self.test_user_name = u'testuser' test_user = self.get_test_user(name=self.test_user_name) From 5ceb68ca095737373abbc301b470a337dd9ddee4 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 22 Jun 2017 01:26:55 +0530 Subject: [PATCH 35/39] fixed typo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8d5d8a9b..42133c0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ python: - 2.7 services: - redis-server -before-script: +before_script: - ./manage.py init # creates the test upload directory script: - ./runtests.sh From bb6a350a60086bd98eb1b8b20b67a994fe954efb Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 22 Jun 2017 01:33:58 +0530 Subject: [PATCH 36/39] setting test environment variable in travis file --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 42133c0b..710a8a9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,12 +13,17 @@ python: - 2.7 services: - redis-server +env: + # this is already being set in runtests.sh, + # but need to set it for `./manage.py init` too. + # could set it before running manage.py, but chose to put it here + - FLASK_ENV=TESTING before_script: - ./manage.py init # creates the test upload directory script: - ./runtests.sh install: - - sudo apt-get -qq install zlib1g-dev libfreetype6-dev liblcms1-dev libwebp-dev libjpeg-dev libpng-dev libfreetype6-dev libtiff4-dev librsvg2-dev ghostscript imagemagick pandoc + - sudo apt-get -qq install zlib1g-dev libfreetype6-dev liblcms1-dev libwebp-dev libjpeg-dev libpng-dev libfreetype6-dev libtiff4-dev librsvg2-dev ghostscript imagemagick pandoc inkscape - pip install -r requirements.txt - pip install -r test_requirements.txt notifications: From 78bd8d3c67c3d4f015e5ac795e4860b0a6e65e1e Mon Sep 17 00:00:00 2001 From: Bibhas Date: Thu, 22 Jun 2017 01:40:07 +0530 Subject: [PATCH 37/39] setting env in existing block, otherwise it gets overwritten --- .travis.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 710a8a9d..25f9730d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,16 +8,15 @@ env: SQ9lTuSD5jg3NSM8yMfMU58ppAYBgd+6VQP01wUPFrV/JSsbfgnVjMTm/Nbk EGeHYvQUiyAg7zM4KdNJr6txj+jBE8MAeh7EwYNHoh9B7Vx//GxmXFnWyjXV 9cJkFroDW1Zfs2SZjLtzMQC8YXE30jmMxg+XHCQewKzRa4u1320= + # this is already being set in runtests.sh, + # but need to set it for `./manage.py init` too. + # could set it before running manage.py, but chose to put it here + - FLASK_ENV=TESTING language: python python: - 2.7 services: - redis-server -env: - # this is already being set in runtests.sh, - # but need to set it for `./manage.py init` too. - # could set it before running manage.py, but chose to put it here - - FLASK_ENV=TESTING before_script: - ./manage.py init # creates the test upload directory script: From 993287c977bb903c422661c54af155c8cf7464b6 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Fri, 23 Jun 2017 09:15:43 +0530 Subject: [PATCH 38/39] moved the init command out of if block --- manage.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/manage.py b/manage.py index 5483a2e8..f1460b6e 100755 --- a/manage.py +++ b/manage.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import os -from coaster.manage import init_manager +from coaster.manage import manager, init_manager from imgee.models import db from imgee import app @@ -11,12 +11,13 @@ def mkdir_p(dirname): if not os.path.exists(dirname): os.makedirs(dirname) + +@manager.command +def init(): + mkdir_p(os.path.join(app.static_folder, app.config['UPLOADED_FILES_DIR'])) + if __name__ == "__main__": db.init_app(app) - manager = init_manager(app, db) - - @manager.command - def init(): - mkdir_p(os.path.join(app.static_folder, app.config['UPLOADED_FILES_DIR'])) + init_manager(app, db) manager.run() From 481701acf7ece0b4c9d7eba2d91769f5f76cc003 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Fri, 23 Jun 2017 10:24:28 +0530 Subject: [PATCH 39/39] rearranged travis config, cleanup of upload directory path --- .travis.yml | 8 ++++---- imgee/__init__.py | 2 ++ imgee/storage.py | 4 ++-- imgee/tasks.py | 4 ++-- imgee/utils.py | 2 +- instance/settings.py | 3 +-- instance/testing.py | 3 ++- manage.py | 2 +- tests/fixtures.py | 5 ----- 9 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 25f9730d..1b11eb41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,14 +17,14 @@ python: - 2.7 services: - redis-server -before_script: - - ./manage.py init # creates the test upload directory -script: - - ./runtests.sh install: - sudo apt-get -qq install zlib1g-dev libfreetype6-dev liblcms1-dev libwebp-dev libjpeg-dev libpng-dev libfreetype6-dev libtiff4-dev librsvg2-dev ghostscript imagemagick pandoc inkscape - pip install -r requirements.txt - pip install -r test_requirements.txt +before_script: + - ./manage.py init # creates the test upload directory +script: + - ./runtests.sh notifications: email: false slack: diff --git a/imgee/__init__.py b/imgee/__init__.py index 6e4efc0d..9b768989 100644 --- a/imgee/__init__.py +++ b/imgee/__init__.py @@ -39,3 +39,5 @@ def error403(error): if app.config.get('MEDIA_DOMAIN', '').lower().startswith(('http://', 'https://')): app.config['MEDIA_DOMAIN'] = app.config['MEDIA_DOMAIN'].split(':', 1)[1] + +app.upload_folder = os.path.join(app.static_folder, app.config.get('UPLOADED_FILES_DIR')) diff --git a/imgee/storage.py b/imgee/storage.py index b2252a3a..fd406389 100644 --- a/imgee/storage.py +++ b/imgee/storage.py @@ -271,7 +271,7 @@ def clean_local_cache(expiry=24): Remove files from local cache which are NOT accessed in the last `expiry` hours. """ - cache_path = os.path.join(app.static_folder, app.config.get('UPLOADED_FILES_DIR')) + cache_path = app.upload_folder cache_path = os.path.join(cache_path, '*') min_atime = time.time() - expiry*60*60 @@ -293,7 +293,7 @@ def delete(stored_file, commit=True): registry.remove_keys_starting_with(stored_file.name) # remove locally - cache_path = os.path.join(app.static_folder, app.config.get('UPLOADED_FILES_DIR')) + cache_path = app.upload_folder os.remove(os.path.join(cache_path, '%s' % stored_file.filename)) cached_img_path = os.path.join(cache_path, '%s_*' % stored_file.name) for f in glob(cached_img_path): diff --git a/imgee/tasks.py b/imgee/tasks.py index 610c360f..46f148db 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -7,11 +7,11 @@ class InvalidRedisQueryException(Exception): class TaskRegistry(object): - def __init__(self, name='default', connection=None, app=None): + def __init__(self, app=None, name='default', connection=None): self.name = name self.connection = connection self.key_prefix = 'imgee:registry:%s' % name - self.filename_pattern = '^[a-z0-9\_\.]+$' + self.filename_pattern = r'^[a-z0-9_\.]+$' if app: self.init_app(app) diff --git a/imgee/utils.py b/imgee/utils.py index 18e7f2ab..2c7246a1 100644 --- a/imgee/utils.py +++ b/imgee/utils.py @@ -97,7 +97,7 @@ def get_file_url(scheme=None): def path_for(img_name): - return os.path.join(app.static_folder, app.config['UPLOADED_FILES_DIR'], img_name) + return os.path.join(app.upload_folder, img_name) # -- mimetypes and content types diff --git a/instance/settings.py b/instance/settings.py index 87ceb3f8..e20edea9 100644 --- a/instance/settings.py +++ b/instance/settings.py @@ -33,8 +33,7 @@ #: Log file LOGFILE = 'error.log' #: File uploads -UPLOADED_FILES_DEST = 'imgee/static/uploads' -UPLOADED_FILES_DIR = 'uploads' # this folder should be inside `imgee/static/` +UPLOADED_FILES_DIR = 'uploads' # this will sit inside `app.static_folder` #: S3 Configuration AWS_ACCESS_KEY = 'Set aws access key here' AWS_SECRET_KEY = 'Set aws secret key here' diff --git a/instance/testing.py b/instance/testing.py index 8b5b1eb2..3a2430be 100644 --- a/instance/testing.py +++ b/instance/testing.py @@ -1,5 +1,6 @@ from os import environ -UPLOADED_FILES_DEST = 'imgee/static/test_uploads' + +# this will sit inside `app.static_folder` UPLOADED_FILES_DIR = 'test_uploads' AWS_FOLDER = 'test/' diff --git a/manage.py b/manage.py index f1460b6e..7c3aea72 100755 --- a/manage.py +++ b/manage.py @@ -14,7 +14,7 @@ def mkdir_p(dirname): @manager.command def init(): - mkdir_p(os.path.join(app.static_folder, app.config['UPLOADED_FILES_DIR'])) + mkdir_p(app.upload_folder) if __name__ == "__main__": db.init_app(app) diff --git a/tests/fixtures.py b/tests/fixtures.py index eb9e1c97..b69eb2e2 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -27,11 +27,6 @@ def setUp(self): app.config['TESTING'] = True app.testing = True - # PYTHONPATH is `pwd` when testing and empty otherwise - # using it to determine the project root - # either get it from environment variable, or it's one level up from this init file - app.project_root = os.environ.get('PYTHONPATH', '') or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - db.create_all() self.test_user_name = u'testuser' test_user = self.get_test_user(name=self.test_user_name)