Repository: steemit/hivemind

Issue:
only first 4 tags are indexed
The symptoms described in this post have been confirmed; PR incoming.

https://steemit.com/steemit/@free-reign/5th-tag-doesn-t-seem-to-be-getting-posts-like-the-other-4-anymore

Comments:

Patch:
diff --git a/hive/indexer/cached_post.py b/hive/indexer/cached_post.py
index 53a6ad0..f731b41 100644
--- a/hive/indexer/cached_post.py
+++ b/hive/indexer/cached_post.py
@@ -470,17 +470,18 @@ class CachedPost:
     @classmethod
     def _tag_sqls(cls, pid, tags, diff=True):
         """Generate SQL "deltas" for a post_id's associated tags."""
+        next_tags = set(tags)
         curr_tags = set()
         if diff:
             sql = "SELECT tag FROM hive_post_tags WHERE post_id = :id"
             curr_tags = set(DB.query_col(sql, id=pid))
 
-        to_rem = (curr_tags - tags)
+        to_rem = (curr_tags - next_tags)
         if to_rem:
             sql = "DELETE FROM hive_post_tags WHERE post_id = :id AND tag IN :tags"
             yield (sql, dict(id=pid, tags=tuple(to_rem)))
 
-        to_add = (tags - curr_tags)
+        to_add = (next_tags - curr_tags)
         if to_add:
             params = _keyify(to_add)
             vals = ["(:id, :%s)" % key for key in params.keys()]
diff --git a/hive/utils/post.py b/hive/utils/post.py
index 562b518..87e5dfe 100644
--- a/hive/utils/post.py
+++ b/hive/utils/post.py
@@ -3,7 +3,7 @@
 
 import math
 import ujson as json
-from funcy.seqs import first
+from funcy.seqs import first, distinct
 
 from hive.utils.normalize import sbd_amount, rep_log10, safe_img_url, parse_time, utc_timestamp
 
@@ -33,8 +33,9 @@ def post_basic(post):
     tags = [post['category']]
     if md and 'tags' in md and isinstance(md['tags'], list):
         tags = tags + md['tags']
-    tags = set(list(map(lambda tag: (str(tag) or '').strip('# ').lower()[:32], tags))[0:5])
-    tags.discard('')
+    tags = map(lambda tag: (str(tag) or '').strip('# ').lower()[:32], tags)
+    tags = filter(None, tags)
+    tags = list(distinct(tags))[:5]
     is_nsfw = 'nsfw' in tags
 
     body = post['body']


Test patch:
diff --git a/tests/utils/test_utils_post.py b/tests/utils/test_utils_post.py
index 58332e9..8c61014 100644
--- a/tests/utils/test_utils_post.py
+++ b/tests/utils/test_utils_post.py
@@ -89,11 +89,59 @@ POST_1 = {
     "vote_rshares": 0
 }
 
+POST_2 = {
+    "abs_rshares": 0,
+    "active": "2017-06-20T15:53:51",
+    "active_votes": [],
+    "allow_curation_rewards": True,
+    "allow_replies": True,
+    "allow_votes": True,
+    "author": "test-safari",
+    "author_reputation": "468237543674",
+    "author_rewards": 23,
+    "beneficiaries": [],
+    "body": "https://pbs.twimg.com/media/DBgNm3jXoAAioyE.jpg",
+    "body_length": 0,
+    "cashout_time": "1969-12-31T23:59:59",
+    "category": "steemit",
+    "children": 0,
+    "children_abs_rshares": 0,
+    "created": "2017-06-20T15:53:51",
+    "curator_payout_value": "0.000 SBD",
+    "depth": 0,
+    "id": 4437869,
+    "json_metadata": "{\"tags\":[\"steemit\",\"steem\",\"\",\"abc\",\"bcd\",\"cde\"]}",
+    "last_payout": "2017-06-27T15:53:51",
+    "last_update": "2017-06-20T15:53:51",
+    "max_accepted_payout": "1000000.000 SBD",
+    "max_cashout_time": "1969-12-31T23:59:59",
+    "net_rshares": 0,
+    "net_votes": 4,
+    "parent_author": "",
+    "parent_permlink": "spam",
+    "pending_payout_value": "0.000 SBD",
+    "percent_steem_dollars": 10000,
+    "permlink": "june-spam",
+    "promoted": "0.000 SBD",
+    "reblogged_by": [],
+    "replies": [],
+    "reward_weight": 10000,
+    "root_author": "test-safari",
+    "root_permlink": "june-spam",
+    "root_title": "June Spam",
+    "title": "June Spam",
+    "total_payout_value": "0.044 SBD",
+    "total_pending_payout_value": "0.000 STEEM",
+    "total_vote_weight": 0,
+    "url": "/spam/@test-safari/june-spam",
+    "vote_rshares": 0
+}
+
 def test_post_basic():
     ret = post_basic(POST_1)
     expect = {'json_metadata': {'tags': ['spam'], 'image': ['https://pbs.twimg.com/media/DBgNm3jXoAAioyE.jpg', 'https://example.com/image.jpg'], 'app': 'steemit/0.1', 'format': 'markdown'},
               'image': 'https://pbs.twimg.com/media/DBgNm3jXoAAioyE.jpg',
-              'tags': {'spam'},
+              'tags': ['spam'],
               'is_nsfw': False,
               'body': 'https://pbs.twimg.com/media/DBgNm3jXoAAioyE.jpg',
               'preview': 'https://pbs.twimg.com/media/DBgNm3jXoAAioyE.jpg',
@@ -103,6 +151,11 @@ def test_post_basic():
               'is_full_power': False}
     assert ret == expect
 
+def test_post_basic_tags():
+    tags = post_basic(POST_2)['tags']
+    expected = ['steemit', 'steem', 'abc', 'bcd', 'cde']
+    assert tags == expected, "got %s" % tags
+
 def test_post_legacy():
     ret = post_legacy(POST_1)
     expect = {'allow_curation_rewards': True,