-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Function name fix, using uuid1mc, time format fix, TaskRegistry fix (#60
) * function name fix, using uuid1mc, time format fix * deleting keys from registry when deleting a file * restructured TaskRegistry to use Redis Keys with TTL set to 60 seconds instead of assigning them to a set * re-adding nose and coverage to travis file, removed it by mistake * using runtests.py file to run tests in travis * reordered functions and better string formating * made the redis key expiry time an argument * assigning error handler the right way * not resizing if the gif image is animated * added pillow as dependency * using redis pipeline to delete multiple keys * missed one place to use redis pipeline * deleted runtests.py, not needed anymore * fixed typo * fixed some import issues * moved upload directory creation to manage.py * added some comments * moved importing app inside the only function that uses it * Update runtests.sh * added manage.py init to runtests * removed mkdir from travis config * fixed upload path issue * moar fixes to registry, cleanup, moved back to uuid4 * fixed typo * setting test environment variable in travis file * setting env in existing block, otherwise it gets overwritten * moved the init command out of if block * rearranged travis config, cleanup of upload directory path
- Loading branch information
Bibhas
authored
Jun 23, 2017
1 parent
cc369e6
commit 380bb87
Showing
14 changed files
with
174 additions
and
79 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,35 +1,85 @@ | ||
| import re | ||
| import redis | ||
| from imgee import app | ||
|
|
||
|
|
||
| class InvalidRedisQueryException(Exception): | ||
| pass | ||
|
|
||
|
|
||
| class TaskRegistry(object): | ||
| def __init__(self, name='default', connection=None): | ||
| self.connection = redis.from_url(connection) if connection else None | ||
| def __init__(self, app=None, name='default', connection=None): | ||
| self.name = name | ||
| self.key = 'imgee:registry:%s' % name | ||
| self.connection = connection | ||
| self.key_prefix = 'imgee:registry:%s' % name | ||
| self.filename_pattern = r'^[a-z0-9_\.]+$' | ||
|
|
||
| def set_connection(self, connection=None): | ||
| connection = connection or app.config.get('REDIS_URL') | ||
| self.connection = redis.from_url(connection) | ||
| if app: | ||
| self.init_app(app) | ||
| else: | ||
| self.app = None | ||
|
|
||
| def add(self, taskid): | ||
| self.connection.sadd(self.key, taskid) | ||
| def init_app(self, app): | ||
| self.app = app | ||
|
|
||
| def remove(self, taskid): | ||
| self.connection.srem(self.key, taskid) | ||
| if not self.connection: | ||
| url = self.app.config.get('REDIS_URL') | ||
| self.connection = redis.from_url(url) | ||
| self.pipe = self.connection.pipeline() | ||
|
|
||
| def remove_all(self): | ||
| for k in self.get_all_keys(): | ||
| self.remove(k) | ||
| def is_valid_query(self, query): | ||
| return bool(re.match(self.filename_pattern, query)) | ||
|
|
||
| def key_for(self, taskid): | ||
| return u'{key_prefix}:{taskid}'.format(key_prefix=self.key_prefix, taskid=taskid) | ||
|
|
||
| def __contains__(self, taskid): | ||
| return self.connection.sismember(self.key, taskid) | ||
| return len(self.connection.keys(self.key_for(taskid))) > 0 | ||
|
|
||
| def keys_starting_with(self, query): | ||
| return filter(lambda k: k.startswith(query), self.connection.smembers(self.key)) | ||
| def add(self, taskid, expire=60): | ||
| self.connection.set(self.key_for(taskid), taskid) | ||
| # setting TTL of 60 seconds for the key | ||
| # if the file doesn't get processed within 60 seconds, | ||
| # it'll be removed from the registry | ||
| self.connection.expire(self.key_for(taskid), expire) | ||
|
|
||
| def search(self, query): | ||
| return filter(lambda k: str(query) in k, self.connection.smembers(self.key)) | ||
| # >> KEYS imgee:registry:default:*query* | ||
| if not self.is_valid_query(query): | ||
| raise InvalidRedisQueryException(u'Invalid query for searching redis keys: {}'.format(query)) | ||
| return self.connection.keys(self.key_for('*{}*'.format(query))) | ||
|
|
||
| def get_all_keys(self): | ||
| return list(self.connection.smembers(self.key)) | ||
| # >> KEYS imgee:registry:default:* | ||
| return self.connection.keys(self.key_for('*')) | ||
|
|
||
| def keys_starting_with(self, query): | ||
| # >> KEYS imgee:registry:default:query_* | ||
| # chances are that we'll use this function to find the | ||
| # thumbnail keys, which look like "name_wNN_hNN", hence the _ | ||
| if not self.is_valid_query(query): | ||
| raise InvalidRedisQueryException(u'Invalid query for searching redis keys, starting with: {}'.format(query)) | ||
| return self.connection.keys(self.key_for('{}_*'.format(query))) | ||
|
|
||
| def remove(self, taskid): | ||
| # remove a key with the taskid | ||
| self.remove_by_key(self.key_for(taskid)) | ||
|
|
||
| def remove_by_key(self, key): | ||
| # remove a redis key directly | ||
| self.connection.delete(key) | ||
|
|
||
| def remove_keys(self, keys): | ||
| for k in keys: | ||
| self.pipe.delete(k) | ||
| self.pipe.execute() | ||
|
|
||
| def remove_keys_matching(self, query): | ||
| self.remove_keys(self.search(query)) | ||
|
|
||
| def remove_keys_starting_with(self, query): | ||
| # query will most likely be stored_file.name, So | ||
| # Delete keys for only `<query>` and `<query>_*` | ||
| self.remove_keys([self.key_for(query)] + self.keys_starting_with(query)) | ||
|
|
||
| def remove_all(self): | ||
| self.remove_keys(self.get_all_keys()) |
Oops, something went wrong.