Repository: lemon24/reader

Issue:
[Question] Public API for recently fetched entries, or entries added/modified since date?
For app I'm working on, I wish to update some feeds and do something on entries that appeared. But I'm unsure how to implement this last part and get only new entries.

Workflow I sketched so far is:
1. Disable updates for all feeds in db (because there might be some I don't wish to update)
2. Start adding feeds to db
2.1. If feed already exists, enable it for updates
2.2. If feed doesn't exist, it will be added and enabled for updates automatically
3. Update feeds through `update_feeds` or `update_feeds_iter`
4. ???

`update_feeds` doesn't return anything. `update_feeds_iter` gives me back list of `UpdateResult`, where each item will have url and counts or exception.

So, I think I can sum all the counts and ask for that many entries. Something like:
```python
count = sum(
    sum(result.value.new, result.value.updated)
    for result in results
    if not isinstance(result.value, ReaderError)
)
new_entries = reader.get_entries(limit=count)
```

But is it guaranteed that `get_entries(sort='recent')` will include recently updated entry? Even if that entry originally appeared long time ago? I might be misunderstanding what it means for entry to be marked as "updated", so any pointer on that would be helpful, too.

Perhaps I could change my workflow a little - first get monotonic timestamp, then run all the steps, and finally ask for all entries that were added or modified after timestamp. But it seems that there is no API for searching by date? `search_entries` is designed for full-text search and works only on few columns.

So, my question is:

1. What is the preferred way of obtaining all entries added in update call?
Counting and using `get_entries`? Calling `get_entries` for everything and discarding all results that were added / modified before timestamp? Something else?
2. What does it mean that entry was "updated"?

Comments:

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index e46cddc..cb3f951 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -11,6 +11,12 @@ Version 1.20
 
 Unreleased
 
+* Added public property :attr:`~Reader.after_entry_update_hooks` to
+  :class:`Reader`, which contains list of hooks to run for each new/modified
+  entry after :meth:`~Reader.update_feeds()` call. Hooks will receive
+  :class:`Reader` instance, :class:`~types.Entry`-like object instance and
+  :class:`~types.EntryUpdateStatus` value
+
 
 Version 1.19
 ------------
diff --git a/docs/api.rst b/docs/api.rst
index 8393dcb..7d757f9 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -61,6 +61,9 @@ Data objects
 .. autoclass:: UpdatedFeed
     :members:
 
+.. autoclass:: EntryUpdateStatus
+    :members:
+
 
 Exceptions
 ----------
diff --git a/src/reader/__init__.py b/src/reader/__init__.py
index 3f8fdaf..690acde 100644
--- a/src/reader/__init__.py
+++ b/src/reader/__init__.py
@@ -54,6 +54,7 @@ from .types import (
     EntrySearchCounts,
     UpdateResult,
     UpdatedFeed,
+    EntryUpdateStatus,
 )
 
 from .exceptions import (
diff --git a/src/reader/core.py b/src/reader/core.py
index cf4bb34..da0e64d 100644
--- a/src/reader/core.py
+++ b/src/reader/core.py
@@ -52,6 +52,7 @@ from .types import EntryInput
 from .types import EntrySearchCounts
 from .types import EntrySearchResult
 from .types import EntrySortOrder
+from .types import EntryUpdateStatus
 from .types import Feed
 from .types import FeedCounts
 from .types import FeedInput
@@ -73,7 +74,12 @@ _U = TypeVar('_U')
 
 ReaderPluginType = Callable[['Reader'], None]
 _PostFeedUpdatePluginType = Callable[['Reader', str], None]
-_PostEntryAddPluginType = Callable[['Reader', EntryData[datetime]], None]
+#: Function that will be called for each new/modified entry
+#:
+#: ..versionadded:: 1.20
+AfterEntryUpdateHook = Callable[
+    ['Reader', EntryData[datetime], EntryUpdateStatus], None
+]
 
 
 def make_reader(
@@ -265,6 +271,9 @@ class Reader:
     .. versionadded:: 1.13
         JSON Feed support.
 
+    .. versionadded:: 1.20
+        after_entry_update_hooks public attribute
+
     """
 
     def __init__(
@@ -286,7 +295,22 @@ class Reader:
 
         self._updater = reader._updater
         self._post_feed_update_plugins: Collection[_PostFeedUpdatePluginType] = []
-        self._post_entry_add_plugins: Collection[_PostEntryAddPluginType] = []
+
+        #: List of functions that will be called for each new/modified entry after
+        #: feed was updated. Each function should take three arguments and return None.
+        #:
+        #: * ``reader`` - a :class:`Reader` instance
+        #: * ``entry`` - an :class:`~types.Entry`-like object
+        #:
+        #:   .. warning::
+        #:
+        #:     The only attributes that are guaranteed to be present are ``feed_url``, ``id``
+        #:     and ``object_id``; all other attributes may appear or disappear between minor
+        #:     versions, or may be None.
+        #: * ``status`` - an :class:`~types.EntryUpdateStatus` value
+        #:
+        #: ..versionadded:: 1.20
+        self.after_entry_update_hooks: Collection[AfterEntryUpdateHook] = []
 
         if _called_directly:
             warnings.warn(
@@ -919,12 +943,12 @@ class Reader:
         for entry in entries_to_update:
             if entry.new:
                 new_count += 1
+                entry_status = EntryUpdateStatus.NEW
             else:
                 updated_count += 1
-            if not entry.new:
-                continue
-            for entry_plugin in self._post_entry_add_plugins:
-                entry_plugin(self, entry.entry)
+                entry_status = EntryUpdateStatus.MODIFIED
+            for entry_hook in self.after_entry_update_hooks:
+                entry_hook(self, entry.entry, entry_status)
 
         return new_count, updated_count
 
diff --git a/src/reader/plugins/entry_dedupe.py b/src/reader/plugins/entry_dedupe.py
index 972401b..91eddb5 100644
--- a/src/reader/plugins/entry_dedupe.py
+++ b/src/reader/plugins/entry_dedupe.py
@@ -74,6 +74,8 @@ Entry user attributes are set as follows:
 import logging
 import re
 
+from reader.types import EntryUpdateStatus
+
 log = logging.getLogger('reader._plugins.feed_entry_dedupe')
 
 
@@ -111,7 +113,10 @@ def _is_duplicate(one, two):
     return same_title and same_text
 
 
-def _entry_dedupe_plugin(reader, entry):
+def _entry_dedupe_plugin(reader, entry, status):
+    if status is EntryUpdateStatus.MODIFIED:
+        return
+
     duplicates = [
         e
         for e in reader.get_entries(feed=entry.feed_url)
@@ -150,4 +155,4 @@ def _entry_dedupe_plugin(reader, entry):
 
 
 def init_reader(reader):
-    reader._post_entry_add_plugins.append(_entry_dedupe_plugin)
+    reader.after_entry_update_hooks.append(_entry_dedupe_plugin)
diff --git a/src/reader/plugins/mark_as_read.py b/src/reader/plugins/mark_as_read.py
index 8f696a7..3b30bf7 100644
--- a/src/reader/plugins/mark_as_read.py
+++ b/src/reader/plugins/mark_as_read.py
@@ -29,6 +29,7 @@ import logging
 import re
 
 from reader.exceptions import MetadataNotFoundError
+from reader.types import EntryUpdateStatus
 
 # avoid circular imports
 
@@ -52,7 +53,10 @@ def _get_config(reader, feed_url, metadata_key, patterns_key):
     return []
 
 
-def _mark_as_read(reader, entry):
+def _mark_as_read(reader, entry, status):
+    if status is EntryUpdateStatus.MODIFIED:
+        return
+
     metadata_name = reader.make_reader_reserved_name('mark_as_read')
     patterns = _get_config(reader, entry.feed_url, metadata_name, 'title')
 
@@ -63,4 +67,4 @@ def _mark_as_read(reader, entry):
 
 
 def init_reader(reader):
-    reader._post_entry_add_plugins.append(_mark_as_read)
+    reader.after_entry_update_hooks.append(_mark_as_read)
diff --git a/src/reader/types.py b/src/reader/types.py
index c3a1f5a..d5c90d5 100644
--- a/src/reader/types.py
+++ b/src/reader/types.py
@@ -1,4 +1,5 @@
 import dataclasses
+import enum
 import re
 import traceback
 import warnings
@@ -480,6 +481,19 @@ class EntrySearchResult(_namedtuple_compat):
         return self.feed_url, self.id
 
 
+class EntryUpdateStatus(enum.Enum):
+
+    """Data type representing status of entry.
+    Used by :py:attr:`~Reader.after_entry_update_hooks`.
+
+    .. versionadded:: 1.20
+
+    """
+
+    NEW = 'new'
+    MODIFIED = 'modified'
+
+
 # Semi-public API (typing support)
 
 


Test patch:
diff --git a/tests/test_plugins_entry_dedupe.py b/tests/test_plugins_entry_dedupe.py
index 11cba18..b11b5b2 100644
--- a/tests/test_plugins_entry_dedupe.py
+++ b/tests/test_plugins_entry_dedupe.py
@@ -92,6 +92,9 @@ def test_plugin(make_reader):
     important_one = parser.entry(
         1, 5, datetime(2010, 1, 1), title='important', summary='also important'
     )
+    modified_one = parser.entry(
+        1, 6, datetime(2010, 1, 1), title='title', summary='will be modified'
+    )
 
     # TODO just use the feeds/entries as arguments
 
@@ -110,6 +113,9 @@ def test_plugin(make_reader):
     important_two = parser.entry(
         1, 15, datetime(2010, 1, 2), title='important', summary='also important'
     )
+    modified_two = parser.entry(
+        1, 6, datetime(2010, 1, 1), title='title', summary='was modified'
+    )
 
     reader.update_feeds()
 
@@ -128,6 +134,8 @@ def test_plugin(make_reader):
             # the old one is marked as read in favor of the new one
             (unread_one.id, True),
             (unread_two.id, False),
+            # modified entry is ignored by plugin
+            (modified_one.id, False),
         }
     } | {
         # the new one is important because the old one was;
diff --git a/tests/test_plugins_mark_as_read.py b/tests/test_plugins_mark_as_read.py
index f8f55d7..cc68e62 100644
--- a/tests/test_plugins_mark_as_read.py
+++ b/tests/test_plugins_mark_as_read.py
@@ -48,10 +48,14 @@ def test_regex_mark_as_read_bad_metadata(make_reader, value):
 
     one = parser.feed(1, datetime(2010, 1, 1))
     parser.entry(1, 1, datetime(2010, 1, 1), title='match')
+    parser.entry(1, 2, datetime(2010, 1, 1), title='will be modified')
 
     reader.add_feed(one)
     reader.set_feed_metadata_item(one, '.reader.mark_as_read', value)
 
     reader.update_feeds()
 
-    assert [e.read for e in reader.get_entries()] == [False]
+    parser.entry(1, 2, datetime(2010, 1, 1), title='modified')
+    reader.update_feeds()
+
+    assert [e.read for e in reader.get_entries()] == [False, False]
diff --git a/tests/test_reader_hooks.py b/tests/test_reader_hooks.py
new file mode 100644
index 0000000..645f903
--- /dev/null
+++ b/tests/test_reader_hooks.py
@@ -0,0 +1,45 @@
+from datetime import datetime
+
+from fakeparser import Parser
+
+from reader.types import EntryUpdateStatus
+
+
+def test_post_entry_update_hooks(reader):
+    parser = Parser()
+    reader._parser = parser
+
+    plugin_calls = []
+
+    def first_plugin(r, e, s):
+        assert r is reader
+        plugin_calls.append((first_plugin, e, s))
+
+    def second_plugin(r, e, s):
+        assert r is reader
+        plugin_calls.append((second_plugin, e, s))
+
+    feed = parser.feed(1, datetime(2010, 1, 1))
+    one = parser.entry(1, 1, datetime(2010, 1, 1))
+    reader.add_feed(feed.url)
+    reader.after_entry_update_hooks.append(first_plugin)
+    reader.update_feeds()
+    assert plugin_calls == [(first_plugin, one, EntryUpdateStatus.NEW)]
+    assert set(e.id for e in reader.get_entries()) == {'1, 1'}
+
+    plugin_calls[:] = []
+
+    feed = parser.feed(1, datetime(2010, 1, 2))
+    one = parser.entry(1, 1, datetime(2010, 1, 2))
+    two = parser.entry(1, 2, datetime(2010, 1, 2))
+    reader.after_entry_update_hooks.append(second_plugin)
+    reader.update_feeds()
+    assert plugin_calls == [
+        (first_plugin, two, EntryUpdateStatus.NEW),
+        (second_plugin, two, EntryUpdateStatus.NEW),
+        (first_plugin, one, EntryUpdateStatus.MODIFIED),
+        (second_plugin, one, EntryUpdateStatus.MODIFIED),
+    ]
+    assert set(e.id for e in reader.get_entries()) == {'1, 1', '1, 2'}
+
+    # TODO: What is the expected behavior if a plugin raises an exception?
diff --git a/tests/test_reader_private.py b/tests/test_reader_private.py
index 545eb5e..fc3f6ac 100644
--- a/tests/test_reader_private.py
+++ b/tests/test_reader_private.py
@@ -95,44 +95,6 @@ def test_update_parse(reader, call_update_method):
     assert parser.calls == [(feed.url, 'etag', 'last-modified')]
 
 
-def test_post_entry_add_plugins(reader):
-    parser = Parser()
-    reader._parser = parser
-
-    plugin_calls = []
-
-    def first_plugin(r, e):
-        assert r is reader
-        plugin_calls.append((first_plugin, e))
-
-    def second_plugin(r, e):
-        assert r is reader
-        plugin_calls.append((second_plugin, e))
-
-    feed = parser.feed(1, datetime(2010, 1, 1))
-    one = parser.entry(1, 1, datetime(2010, 1, 1))
-    reader.add_feed(feed.url)
-    reader._post_entry_add_plugins.append(first_plugin)
-    reader.update_feeds()
-    assert plugin_calls == [(first_plugin, one)]
-    assert set(e.id for e in reader.get_entries()) == {'1, 1'}
-
-    plugin_calls[:] = []
-
-    feed = parser.feed(1, datetime(2010, 1, 2))
-    one = parser.entry(1, 1, datetime(2010, 1, 2))
-    two = parser.entry(1, 2, datetime(2010, 1, 2))
-    reader._post_entry_add_plugins.append(second_plugin)
-    reader.update_feeds()
-    assert plugin_calls == [
-        (first_plugin, two),
-        (second_plugin, two),
-    ]
-    assert set(e.id for e in reader.get_entries()) == {'1, 1', '1, 2'}
-
-    # TODO: What is the expected behavior if a plugin raises an exception?
-
-
 def test_post_feed_update_plugins(reader):
     parser = Parser()
     reader._parser = parser