Merge pull request #156 from steemit/condenser-prep

condenser prep

.circleci/config.yml

@@ -0,0 +1,14 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: docker:17
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run: docker build .
+workflows:
+  version: 2
+  build:
+    jobs:
+      - build

docs/api-examples.md

@@ -27,6 +27,11 @@ http -j post http://localhost:8080 jsonrpc=2.0 id=1 method=condenser_api.get_con
 
 http -j post http://localhost:8080 jsonrpc=2.0 id=1 method=condenser_api.get_state params:='{"path":"spam/@test-safari/34gfex-december-spam"}'
 http -j post http://localhost:8080 jsonrpc=2.0 id=1 method=condenser_api.get_state params:='{"path":"trending"}'
+
+http -j post http://localhost:8080 jsonrpc=2.0 id=1 method=condenser_api.get_discussions_by_author_before_date params:='["test-safari","","2128-03-20T20:27:30",10]'
+http -j post http://localhost:8080 jsonrpc=2.0 id=1 method=condenser_api.get_blog params:='["test-safari", 5, 3]'
+http -j post http://localhost:8080 jsonrpc=2.0 id=1 method=condenser_api.get_blog_entries params:='["test-safari", 5, 3]'
+
 ```
 
 

hive/cli.py

@@ -13,7 +13,7 @@ def run():
 
     conf = Conf.init_argparse()
     Db.set_shared_instance(conf.db())
-    mode = '/'.join(conf.get('mode'))
+    mode = conf.mode()
 
     if mode == 'server':
         from hive.server.serve import run_server

hive/conf.py

@@ -6,6 +6,7 @@
 from hive.steem.client import SteemClient
 from hive.db.adapter import Db
 from hive.utils.normalize import strtobool, int_log_level
+from hive.utils.stats import Stats
 
 class Conf():
     """ Manages sync/server configuration via args, ENVs, and hive.conf. """
@@ -52,6 +53,10 @@ def init_argparse(cls, strict=True, **kwargs):
         root.info("loaded configuration:\n%s",
                   parser.format_values())
 
+        # for API server, dump SQL report often
+        if conf.mode() == 'server':
+            Stats.PRINT_THRESH_MINS = 1
+
         return conf
 
     @classmethod
@@ -92,6 +97,15 @@ def get(self, param):
         assert self._args, "run init_argparse()"
         return self._args[param]
 
+    def mode(self):
+        """Get the CLI runmode.
+
+        - `server`: API server
+        - `sync`: db sync process
+        - `status`: status info dump
+        """
+        return '/'.join(self.get('mode'))
+
     def log_level(self):
         """Get `logger`s internal int level from config string."""
         return int_log_level(self.get('log_level'))

hive/server/condenser_api/call.py

@@ -1,8 +1,8 @@
 """Handles legacy `call` method."""
 
+from hive.server.condenser_api.common import ApiError
 from hive.server.condenser_api.get_state import get_state
 from hive.server.condenser_api.tags import get_trending_tags
-
 from hive.server.condenser_api.methods import (
     get_followers,
     get_following,
@@ -16,19 +16,28 @@
     get_discussions_by_blog,
     get_discussions_by_feed,
     get_discussions_by_comments,
-    get_replies_by_last_update)
+    get_replies_by_last_update,
+
+    get_discussions_by_author_before_date,
+    get_blog,
+    get_blog_entries,
+)
 
-def _strict_list(params, expected_len):
+def _strict_list(params, expected_len, min_len = None):
     assert isinstance(params, list), "params not a list"
-    assert len(params) == expected_len, "expected %d params" % expected_len
+    if min_len is None:
+        assert len(params) == expected_len, "expected %d params" % expected_len
+    else:
+        assert (len(params) <= expected_len and
+                len(params) >= min_len), "expected %d params" % expected_len
     return params
 
 def _strict_query(params, ignore_key=None):
     query = _strict_list(params, 1)[0]
     assert isinstance(query, dict), "query must be dict"
 
-    optional_keys = set(['truncate_body'])
-    expected_keys = set(['start_author', 'start_permlink', 'limit', 'tag'])
+    optional_keys = set(['truncate_body', 'start_author', 'start_permlink'])
+    expected_keys = set(['limit', 'tag'])
     if ignore_key: # e.g. `tag` unused by get_discussion_by_comments
         expected_keys = expected_keys - set([ignore_key])
 
@@ -93,4 +102,12 @@ async def call(api, method, params):
     elif method == 'get_replies_by_last_update':
         return await get_replies_by_last_update(*_strict_list(params, 3))
 
-    raise Exception("unknown method: {}.{}({})".format(api, method, params))
+    # Exotic account discussion queries
+    elif method == 'get_discussions_by_author_before_date':
+        return await get_discussions_by_author_before_date(*_strict_list(params, 4))
+    elif method == 'get_blog':
+        return await get_blog(*_strict_list(params, 3, 2))
+    elif method == 'get_blog_entries':
+        return await get_blog_entries(*_strict_list(params, 3, 2))
+
+    raise ApiError("unknown method: %s.%s" % (api, method))

hive/server/condenser_api/common.py

@@ -1,37 +1,64 @@
 """Helpers for condenser_api calls."""
 
 import re
+from functools import wraps
 
 from hive.db.methods import query_one, query_col
 
+class ApiError(Exception):
+    """API-specific errors: unimplemented/bad params. Pass back to client."""
+    pass
+
+def return_error_info(function):
+    """Async API method decorator which catches and formats exceptions."""
+    @wraps(function)
+    async def wrapper(*args, **kwargs):
+        """Catch ApiError and AssersionError (always due to user error)."""
+        try:
+            return await function(*args, **kwargs)
+        except (ApiError, AssertionError) as e:
+            return {
+                "error": {
+                    "code": -32000,
+                    "message": str(e) + " (hivemind-alpha)"}}
+    return wrapper
+
 def valid_account(name, allow_empty=False):
     """Returns validated account name or throws Assert."""
+    if not name:
+        assert allow_empty, 'account must be specified'
+        return ""
     assert isinstance(name, str), "account must be string; received: %s" % name
-    if not (allow_empty and name == ''):
-        assert len(name) >= 3 and len(name) <= 16, "invalid account: %s" % name
-        assert re.match(r'^[a-z0-9-\.]+$', name), 'invalid account char'
+    assert len(name) >= 3 and len(name) <= 16, "invalid account: %s" % name
+    assert re.match(r'^[a-z0-9-\.]+$', name), 'invalid account char'
     return name
 
 def valid_permlink(permlink, allow_empty=False):
     """Returns validated permlink or throws Assert."""
-    assert isinstance(permlink, str), "permlink must be string: %s" % permlink
-    if not (allow_empty and permlink == ''):
-        assert permlink and len(permlink) <= 256, "invalid permlink"
+    if not permlink:
+        assert allow_empty, 'permlink cannot be blank'
+        return ""
+    assert isinstance(permlink, str), 'permlink must be string'
+    assert len(permlink) <= 256, "invalid permlink length"
     return permlink
 
 def valid_sort(sort, allow_empty=False):
     """Returns validated sort name or throws Assert."""
+    if not sort:
+        assert allow_empty, 'sort must be specified'
+        return ""
     assert isinstance(sort, str), 'sort must be a string'
-    if not (allow_empty and sort == ''):
-        valid_sorts = ['trending', 'promoted', 'hot', 'created']
-        assert sort in valid_sorts, 'invalid sort'
+    valid_sorts = ['trending', 'promoted', 'hot', 'created']
+    assert sort in valid_sorts, 'invalid sort'
     return sort
 
 def valid_tag(tag, allow_empty=False):
     """Returns validated tag or throws Assert."""
+    if not tag:
+        assert allow_empty, 'tag was blank'
+        return ""
     assert isinstance(tag, str), 'tag must be a string'
-    if not (allow_empty and tag == ''):
-        assert re.match('^[a-z0-9-]+$', tag), 'invalid tag'
+    assert re.match('^[a-z0-9-]+$', tag), 'invalid tag'
     return tag
 
 def valid_limit(limit, ubound=100):
@@ -41,6 +68,14 @@ def valid_limit(limit, ubound=100):
     assert limit <= ubound, "limit exceeds max"
     return limit
 
+def valid_offset(offset, ubound=None):
+    """Given a user-provided offset, return a valid int, or raise."""
+    offset = int(offset)
+    assert offset >= 0, "offset cannot be negative"
+    if ubound is not None:
+        assert offset <= ubound, "offset too large"
+    return offset
+
 def get_post_id(author, permlink):
     """Given an author/permlink, retrieve the id from db."""
     sql = ("SELECT id FROM hive_posts WHERE author = :a "

hive/server/condenser_api/cursor.py

@@ -1,16 +1,24 @@
 """Cursor-based pagination queries, mostly supporting condenser_api."""
 
+from datetime import datetime
+from dateutil.relativedelta import relativedelta
+
 from hive.db.methods import query_one, query_col, query_row, query_all
 
+def last_month():
+    """Get the date 1 month ago."""
+    return datetime.now() + relativedelta(months=-1)
+
 def _get_post_id(author, permlink):
     """Get post_id from hive db."""
     sql = "SELECT id FROM hive_posts WHERE author = :a AND permlink = :p"
     return query_one(sql, a=author, p=permlink)
 
 def _get_account_id(name):
     """Get account id from hive db."""
+    assert name, 'no account name specified'
     _id = query_one("SELECT id FROM hive_accounts WHERE name = :n", n=name)
-    assert _id, "invalid account `%s`" % name
+    assert _id, "account `%s` not found" % name
     return _id
 
 
@@ -21,7 +29,7 @@ def get_followers(account: str, start: str, state: int, limit: int):
 
     sql = """
         SELECT name FROM hive_follows hf
-          JOIN hive_accounts ON hf.follower = id
+     LEFT JOIN hive_accounts ON hf.follower = id
          WHERE hf.following = :account_id
            AND state = :state %s
       ORDER BY name ASC
@@ -38,7 +46,7 @@ def get_following(account: str, start: str, state: int, limit: int):
 
     sql = """
         SELECT name FROM hive_follows hf
-          JOIN hive_accounts ON hf.following = id
+     LEFT JOIN hive_accounts ON hf.following = id
          WHERE hf.follower = :account_id
            AND state = :state %s
       ORDER BY name ASC
@@ -130,6 +138,58 @@ def pids_by_blog(account: str, start_author: str = '',
     return query_col(sql, account_id=account_id, limit=limit)
 
 
+def pids_by_blog_by_index(account: str, start_index: int, limit: int = 20):
+    """Get post_ids for an author's blog (w/ reblogs), paged by index/limit.
+
+    Examples:
+    (acct, 2) = returns blog entries 0 up to 2 (3 oldest)
+    (acct, 0) = returns all blog entries (limit 0 means return all?)
+    (acct, 2, 1) = returns 1 post starting at idx 2
+    (acct, 2, 3) = returns 3 posts: idxs (2,1,0)
+    """
+
+
+    sql = """
+        SELECT post_id
+          FROM hive_feed_cache
+         WHERE account_id = :account_id
+      ORDER BY created_at
+         LIMIT :limit
+        OFFSET :offset
+    """
+
+    account_id = _get_account_id(account)
+
+    offset = start_index - limit + 1
+    assert offset >= 0, 'start_index and limit combination is invalid'
+
+    ids = query_col(sql, account_id=account_id, limit=limit, offset=offset)
+    return list(reversed(ids))
+
+
+def pids_by_blog_without_reblog(account: str, start_permlink: str = '', limit: int = 20):
+    """Get a list of post_ids for an author's blog without reblogs."""
+
+    seek = ''
+    if start_permlink:
+        start_id = _get_post_id(account, start_permlink)
+        if not start_id:
+            return []
+        seek = "AND id <= %d" % start_id
+
+    sql = """
+        SELECT id
+          FROM hive_posts
+         WHERE author = :account %s
+           AND is_deleted = '0'
+           AND depth = 0
+      ORDER BY id DESC
+         LIMIT :limit
+    """ % seek
+
+    return query_col(sql, account=account, limit=limit)
+
+
 def pids_by_feed_with_reblog(account: str, start_author: str = '',
                              start_permlink: str = '', limit: int = 20):
     """Get a list of [post_id, reblogged_by_str] for an account's feed."""
@@ -154,11 +214,12 @@ def pids_by_feed_with_reblog(account: str, start_author: str = '',
           JOIN hive_follows ON account_id = hive_follows.following AND state = 1
           JOIN hive_accounts ON hive_follows.following = hive_accounts.id
          WHERE hive_follows.follower = :account
+           AND hive_feed_cache.created_at > :cutoff
       GROUP BY post_id %s
       ORDER BY MIN(hive_feed_cache.created_at) DESC LIMIT :limit
     """ % seek
 
-    return query_all(sql, account=account_id, limit=limit)
+    return query_all(sql, account=account_id, limit=limit, cutoff=last_month())
 
 
 def pids_by_account_comments(account: str, start_permlink: str = '', limit: int = 20):