From e7f46fae57543ef3b1c77c17f3b190d60a53ea22 Mon Sep 17 00:00:00 2001 From: Bibhas Date: Wed, 21 Jun 2017 15:51:41 +0530 Subject: [PATCH] moar fixes to registry, cleanup, moved back to uuid4 --- .travis.yml | 2 ++ imgee/__init__.py | 14 +++----------- imgee/storage.py | 4 ++-- imgee/tasks.py | 26 ++++++++++++++------------ imgee/utils.py | 6 +++--- instance/settings.py | 1 + instance/testing.py | 2 ++ manage.py | 2 +- runtests.sh | 3 +-- tests/fixtures.py | 7 +++++++ 10 files changed, 36 insertions(+), 31 deletions(-) diff --git a/.travis.yml b/.travis.yml index a7aba76f..8d5d8a9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ python: - 2.7 services: - redis-server +before-script: + - ./manage.py init # creates the test upload directory script: - ./runtests.sh install: diff --git a/imgee/__init__.py b/imgee/__init__.py index a90e4740..6e4efc0d 100644 --- a/imgee/__init__.py +++ b/imgee/__init__.py @@ -24,26 +24,18 @@ registry = TaskRegistry() -# Configure the app +# Configure the application coaster.app.init_app(app) migrate = Migrate(app, db) baseframe.init_app(app, requires=['baseframe', 'picturefill', 'imgee']) lastuser.init_app(app) lastuser.init_usermanager(UserManager(db, models.User)) -registry.init_app() - -# PYTHONPATH is `pwd` when testing and empty otherwise -# using it to determine the project root -# either get it from environment variable, or it's one level up from this init file -app.project_root = os.environ.get('PYTHONPATH', '') or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -print app.project_root +registry.init_app(app) @app.errorhandler(403) def error403(error): return redirect(url_for('login')) -if app.config.get('MEDIA_DOMAIN') and ( - app.config['MEDIA_DOMAIN'].startswith('http:') or - app.config['MEDIA_DOMAIN'].startswith('https:')): +if app.config.get('MEDIA_DOMAIN', '').lower().startswith(('http://', 'https://')): app.config['MEDIA_DOMAIN'] = app.config['MEDIA_DOMAIN'].split(':', 1)[1] diff --git a/imgee/storage.py b/imgee/storage.py index 9a4c2aa9..b2252a3a 100644 --- a/imgee/storage.py +++ b/imgee/storage.py @@ -271,7 +271,7 @@ def clean_local_cache(expiry=24): Remove files from local cache which are NOT accessed in the last `expiry` hours. """ - cache_path = app.config.get('UPLOADED_FILES_DEST') + cache_path = os.path.join(app.static_folder, app.config.get('UPLOADED_FILES_DIR')) cache_path = os.path.join(cache_path, '*') min_atime = time.time() - expiry*60*60 @@ -293,7 +293,7 @@ def delete(stored_file, commit=True): registry.remove_keys_starting_with(stored_file.name) # remove locally - cache_path = os.path.join(app.project_root, app.config.get('UPLOADED_FILES_DEST')) + cache_path = os.path.join(app.static_folder, app.config.get('UPLOADED_FILES_DIR')) os.remove(os.path.join(cache_path, '%s' % stored_file.filename)) cached_img_path = os.path.join(cache_path, '%s_*' % stored_file.name) for f in glob(cached_img_path): diff --git a/imgee/tasks.py b/imgee/tasks.py index 70483fdc..610c360f 100644 --- a/imgee/tasks.py +++ b/imgee/tasks.py @@ -8,25 +8,27 @@ class InvalidRedisQueryException(Exception): class TaskRegistry(object): def __init__(self, name='default', connection=None, app=None): - if app: - self.init_app(name, connection) - - def init_app(self, name='default', connection=None): - self.connection = connection or self.set_connection_from_url() self.name = name + self.connection = connection self.key_prefix = 'imgee:registry:%s' % name self.filename_pattern = '^[a-z0-9\_\.]+$' - self.pipe = self.connection.pipeline() + + if app: + self.init_app(app) + else: + self.app = None + + def init_app(self, app): + self.app = app + + if not self.connection: + url = self.app.config.get('REDIS_URL') + self.connection = redis.from_url(url) + self.pipe = self.connection.pipeline() def is_valid_query(self, query): return bool(re.match(self.filename_pattern, query)) - def set_connection_from_url(self, url=None): - from imgee import app - url = url or app.config.get('REDIS_URL') - self.connection = redis.from_url(url) - return self.connection - def key_for(self, taskid): return u'{key_prefix}:{taskid}'.format(key_prefix=self.key_prefix, taskid=taskid) diff --git a/imgee/utils.py b/imgee/utils.py index 7bb0a1da..18e7f2ab 100644 --- a/imgee/utils.py +++ b/imgee/utils.py @@ -13,7 +13,7 @@ import magic from PIL import Image -from coaster.utils import uuid1mc +from uuid import uuid4 from imgee import app THUMBNAIL_COMMANDS = { @@ -84,7 +84,7 @@ def newid(): - return unicode(uuid1mc().hex) + return unicode(uuid4().hex) def get_media_domain(scheme=None): @@ -97,7 +97,7 @@ def get_file_url(scheme=None): def path_for(img_name): - return os.path.join(app.project_root, app.config['UPLOADED_FILES_DEST'], img_name) + return os.path.join(app.static_folder, app.config['UPLOADED_FILES_DIR'], img_name) # -- mimetypes and content types diff --git a/instance/settings.py b/instance/settings.py index 161fc352..87ceb3f8 100644 --- a/instance/settings.py +++ b/instance/settings.py @@ -34,6 +34,7 @@ LOGFILE = 'error.log' #: File uploads UPLOADED_FILES_DEST = 'imgee/static/uploads' +UPLOADED_FILES_DIR = 'uploads' # this folder should be inside `imgee/static/` #: S3 Configuration AWS_ACCESS_KEY = 'Set aws access key here' AWS_SECRET_KEY = 'Set aws secret key here' diff --git a/instance/testing.py b/instance/testing.py index 54a6cf84..8b5b1eb2 100644 --- a/instance/testing.py +++ b/instance/testing.py @@ -1,5 +1,7 @@ from os import environ UPLOADED_FILES_DEST = 'imgee/static/test_uploads' +UPLOADED_FILES_DIR = 'test_uploads' + AWS_FOLDER = 'test/' UNKNOWN_FILE_THUMBNAIL = 'unknown.jpeg' diff --git a/manage.py b/manage.py index 3c121259..5483a2e8 100755 --- a/manage.py +++ b/manage.py @@ -17,6 +17,6 @@ def mkdir_p(dirname): @manager.command def init(): - mkdir_p(os.path.join(app.project_root, app.config['UPLOADED_FILES_DEST'])) + mkdir_p(os.path.join(app.static_folder, app.config['UPLOADED_FILES_DIR'])) manager.run() diff --git a/runtests.sh b/runtests.sh index e105d6c1..37ced6f7 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,7 +1,6 @@ #!/bin/sh set -e export FLASK_ENV="TESTING" -export PYTHONPATH=$(pwd) # setting project root as PYTHONPATH -./manage.py init # creates the test upload directory +export PYTHONPATH=$(dirname $0) # setting project root as PYTHONPATH coverage run `which nosetests` "$@" coverage report -m diff --git a/tests/fixtures.py b/tests/fixtures.py index e5aad2b6..eb9e1c97 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,3 +1,4 @@ +import os import unittest import random import string @@ -25,6 +26,12 @@ def get_test_profile(self): def setUp(self): app.config['TESTING'] = True app.testing = True + + # PYTHONPATH is `pwd` when testing and empty otherwise + # using it to determine the project root + # either get it from environment variable, or it's one level up from this init file + app.project_root = os.environ.get('PYTHONPATH', '') or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + db.create_all() self.test_user_name = u'testuser' test_user = self.get_test_user(name=self.test_user_name)