class CommentTreePermacache(object):
@classmethod
def _permacache_key(cls, link):
return 'comments_' + str(link._id)
@classmethod
def _mutation_context(cls, link):
"""Return a lock for use during read-modify-write operations"""
key = 'comment_lock_' + str(link._id)
return g.make_lock("comment_tree", key)
@classmethod
def prepare_new_storage(cls, link):
"""Write an empty tree to permacache"""
with cls._mutation_context(link) as lock:
# read-modify-write, so get the lock
existing_tree = cls._load_tree(link)
if not existing_tree:
# don't overwrite an existing non-empty tree
tree = {}
cls._write_tree(link, tree, lock)
@classmethod
def _load_tree(cls, link):
key = cls._permacache_key(link)
tree = g.permacache.get(key)
return tree or {} # assume empty tree on miss
@classmethod
def _write_tree(cls, link, tree, lock):
assert lock.have_lock
key = cls._permacache_key(link)
g.permacache.set(key, tree)
@classmethod
def get_tree_pieces(cls, link, timer):
tree = cls._load_tree(link)
timer.intermediate('load')
cids, depth, parents = get_tree_details(tree)
num_children = calc_num_children(tree)
num_children = defaultdict(int, num_children)
timer.intermediate('calculate')
return cids, tree, depth, parents, num_children
@classmethod
def add_comments(cls, link, comments):
with cls._mutation_context(link) as lock:
# adding comments requires read-modify-write, so get the lock
tree = cls._load_tree(link)
cids, _, _ = get_tree_details(tree)
# skip any comments that are already in the stored tree and convert
# to a set to remove any duplicate comments
comments = {
comment for comment in comments
if comment._id not in cids
}
if not comments:
return
# warn on any comments whose parents are missing from the tree
# because they will never be displayed unless their parent is
# added. this can happen in normal operation if there are multiple
# queue consumers and a child is processed before its parent.
parent_ids = set(cids) | {comment._id for comment in comments}
possible_orphan_comments = {
comment for comment in comments
if (comment.parent_id and comment.parent_id not in parent_ids)
}
if possible_orphan_comments:
g.log.error("comment_tree_inconsistent: %s %s", link,
possible_orphan_comments)
g.stats.simple_event('comment_tree_inconsistent')
for comment in comments:
tree.setdefault(comment.parent_id, []).append(comment._id)
cls._write_tree(link, tree, lock)
@classmethod
def rebuild(cls, link, comments):
"""Generate a tree from comments and overwrite any existing tree."""
with cls._mutation_context(link) as lock:
# not reading, but we should block other read-modify-write
# operations to avoid being clobbered by their write
tree = {}
for comment in comments:
tree.setdefault(comment.parent_id, []).append(comment._id)
cls._write_tree(link, tree, lock)
class CommentTree:
def __init__(self, link, cids, tree, depth, parents, num_children):
self.link = link
self.cids = cids
self.tree = tree
self.depth = depth
self.parents = parents
self.num_children = num_children
@classmethod
def by_link(cls, link, timer=None):
if timer is None:
timer = SimpleSillyStub()
pieces = CommentTreePermacache.get_tree_pieces(link, timer)
cids, tree, depth, parents, num_children = pieces
comment_tree = cls(link, cids, tree, depth, parents, num_children)
return comment_tree
@classmethod
def on_new_link(cls, link):
CommentTreePermacache.prepare_new_storage(link)
@classmethod
def add_comments(cls, link, comments):
CommentTreePermacache.add_comments(link, comments)
@classmethod
def rebuild(cls, link):
# retrieve all the comments for the link
q = Comment._query(
Comment.c.link_id == link._id,
Comment.c._deleted == (True, False),
Comment.c._spam == (True, False),
optimize_rules=True,
)
comments = list(q)
# remove any comments with missing parents
comment_ids = {comment._id for comment in comments}
comments = [
comment for comment in comments
if not comment.parent_id or comment.parent_id in comment_ids
]
CommentTreePermacache.rebuild(link, comments)
link.num_comments = sum(1 for c in comments if not c._deleted)
link._commit()
