Repository: lokal-profil/runeberg

Issue:
Add text prop to work and article
Add a `text` property (or similar) to the Work and Article classes which returns the ocred text of the included pages. Once found the text should be stored in e.g. `_text` for subsequent use.

For Article this should respect any `<chapter>`/`</chapter>` limits.
Rename page.ocr
If #18 is implemented, consider renaming `page.ocr` to `page.text` for uniformity.

Discussion:

Patch:
diff --git a/runeberg/article.py b/runeberg/article.py
index 671fa87..c444f84 100644
--- a/runeberg/article.py
+++ b/runeberg/article.py
@@ -16,6 +16,7 @@ advertisement might interrupt it.
 Note that some entries in Articles.lst are only used as Table of Content
 headers, as such they should not be treated as proper articles.
 """
+# @TODO: Handle sub-articles. See e.g. "register" http://runeberg.org/aoshbok/
 from bs4 import BeautifulSoup
 
 from runeberg.page_range import PageRange
@@ -84,6 +85,11 @@ class Article(object):
             self._clean_title = text.strip()
         return self._clean_title
 
+    @property
+    def text(self):
+        """Return all the text of the article."""
+        return '\n'.join([page.text for page in self.pages])
+
     @property
     def uid(self):
         """Work-unique title/id for the article."""
diff --git a/runeberg/page.py b/runeberg/page.py
index 604c251..4f1666e 100644
--- a/runeberg/page.py
+++ b/runeberg/page.py
@@ -15,7 +15,7 @@ class Page(object):
     BLANK_LABELS = ('blank', '(blank)')  # labels known to indicate blank page
     DEFAULT_BLANK = '<blank>'  # default label to use for blank pages
 
-    def __init__(self, uid, label=None, image=None, ocr=None, proofread=None):
+    def __init__(self, uid, label=None, image=None, text=None, proofread=None):
         """
         Initialise a Page.
 
@@ -24,14 +24,14 @@ class Page(object):
         @param label: the real printable label of the page, ordinarily the page
             number.
         @param image: path to the image of the page
-        @param ocr: ocr:ed (and potentially proofread) text of the page
+        @param text: ocr:ed (and potentially proofread) text of the page
         @param proofread: whether the page has been fully proofread.
         """
         self.uid = uid  # unique identifier of the page (often a padded number)
         self.label = label or ''  # what the numbering on the actual page is
         self.image = image or ''  # path to the image
         self.image_no = None  # page in djvu file on which the image appears
-        self.ocr = ocr or ''  # ocr:ed text of the page
+        self.text = text or ''  # ocr:ed text of the page
         self.is_proofread = proofread  # bool or None in the case of blank page
         self.set_blank()
 
@@ -61,19 +61,19 @@ class Page(object):
 
     def check_blank(self):
         """Sanity check blank pages and blank page candidates."""
-        if self.is_proofread is None and self.ocr.strip() != '':
+        if self.is_proofread is None and self.text.strip() != '':
             warnings.warn(
-                '{0} ({1}) was labelled "blank" but ocr is not empty. Either '
+                '{0} ({1}) was labelled "blank" but text is not empty. Either '
                 'clear out the ocr text or set `is_proofread=False`'.format(
                     self.uid, self.label),
                 UserWarning)
-        elif self.is_proofread and self.ocr.strip() == '':
+        elif self.is_proofread and self.text.strip() == '':
             warnings.warn(
                 '{0} ({1}) was labelled as proofread but is blank. Either '
                 're-add the ocr text or set `is_proofread=None`'.format(
                     self.uid, self.label),
                 UserWarning)
-        elif self.is_proofread == False and self.ocr.strip() == '':
+        elif self.is_proofread is False and self.text.strip() == '':
             warnings.warn(
                 '{0} ({1}) might be blank. Check the image and if truly blank '
                 'set `is_proofread=None`'.format(
@@ -111,7 +111,7 @@ class Page(object):
     def get_chapters(self):
         """Extract all chapter names on page."""
         if not hasattr(self, '_chapters'):
-            soup = BeautifulSoup(self.ocr, features='html5lib')
+            soup = BeautifulSoup(self.text, features='html5lib')
             self._chapters = [chapter.get('name')
                               for chapter in soup.find_all('chapter')]
             if any(not chapter for chapter in self._chapters):
@@ -123,7 +123,7 @@ class Page(object):
     def rename_chapter(self, old_name, new_name):
         """Rename the first occurrence of a chapter."""
         base_str = '<chapter name="{}">'
-        self.ocr = self.ocr.replace(
+        self.text = self.text.replace(
             base_str.format(old_name),
             base_str.format(new_name),
             1)
diff --git a/runeberg/page_range.py b/runeberg/page_range.py
index 5d0891c..a0b095b 100644
--- a/runeberg/page_range.py
+++ b/runeberg/page_range.py
@@ -49,6 +49,11 @@ class PageRange(OrderedDict):
         else:
             return '{}-{}'.format(key_list[0], key_list[-1])
 
+    @property
+    def text(self):
+        """Return the text from all the pages in the range."""
+        return '\n'.join([page.text for page in self.values()])
+
     # not implemented in __getitem__ to avoid it mixing keys and indexes
     # at the cost of a bulkier syntax
     def slice(self, start=None, stop=None, step=None):
diff --git a/runeberg/work.py b/runeberg/work.py
index 9cefb1f..beddbc8 100644
--- a/runeberg/work.py
+++ b/runeberg/work.py
@@ -72,6 +72,11 @@ class Work(object):
         work.parse_people(known_people)
         return work
 
+    @property
+    def text(self):
+        """Return all the text of the work."""
+        return self.pages.text
+
     def runeberg_url(self):
         """Return the base url on runeberg.org."""
         return '{0}/{1}/'.format(SITE, self.uid)


Test patch:
diff --git a/tests/test_article.py b/tests/test_article.py
index c0dd108..cfde9a2 100644
--- a/tests/test_article.py
+++ b/tests/test_article.py
@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 """Unit tests for article."""
 import unittest
+import unittest.mock as mock
 
 from runeberg.article import Article
 from runeberg.page import Page
+from runeberg.page_range import PageRange
 
 
 class TestIsTocHeader(unittest.TestCase):
@@ -49,3 +51,35 @@ class TestUid(unittest.TestCase):
     def test_uid_with_disambig(self):
         self.article.disambig = 2
         self.assertEqual(self.article.uid, 'Genesis (2)')
+
+
+class TestText(unittest.TestCase):
+
+    def setUp(self):
+        self.article = Article('Genesis')
+        # cannot get instance attribute mocking to work for page
+        self.page = Page('page_1', text='This is the page text.')
+        self.page_range = PageRange([])
+
+        patcher = mock.patch('runeberg.page_range.PageRange.text',
+                             new_callable=mock.PropertyMock)
+        self.mock_page_range_text = patcher.start()
+        self.mock_page_range_text.return_value = 'This is the page_range text.'
+        self.addCleanup(patcher.stop)
+
+    def test_text_no_pages(self):
+        self.assertEqual(self.article.text, '')
+
+    def test_text_page(self):
+        self.article.pages = [self.page]
+        self.assertEqual(self.article.text, 'This is the page text.')
+
+    def test_text_page_range(self):
+        self.article.pages = [self.page_range]
+        self.assertEqual(self.article.text, 'This is the page_range text.')
+
+    def test_text_multiple_pages(self):
+        self.article.pages = [self.page, self.page_range]
+        self.assertEqual(
+            self.article.text,
+            'This is the page text.\nThis is the page_range text.')
diff --git a/tests/test_page.py b/tests/test_page.py
index 11ba190..0265795 100644
--- a/tests/test_page.py
+++ b/tests/test_page.py
@@ -24,11 +24,11 @@ class TestGetChapters(unittest.TestCase):
         self.page = Page('0001')
 
     def test_get_chapters_empty(self):
-        self.page.ocr = ''
+        self.page.text = ''
         self.assertEqual(self.page.get_chapters(), [])
 
     def test_get_chapters_no_chapters(self):
-        self.page.ocr = (
+        self.page.text = (
             'row 1\n'
             'row 2\n'
             'row3'
@@ -36,7 +36,7 @@ class TestGetChapters(unittest.TestCase):
         self.assertEqual(self.page.get_chapters(), [])
 
     def test_get_chapters_unique_chapters(self):
-        self.page.ocr = (
+        self.page.text = (
             'row 1\n'
             'pre<chapter name="chp1">post\n'
             'row 2\n'
@@ -47,7 +47,7 @@ class TestGetChapters(unittest.TestCase):
         self.assertEqual(self.page.get_chapters(), expected)
 
     def test_get_chapters_non_unique_chapters(self):
-        self.page.ocr = (
+        self.page.text = (
             'row 1\n'
             'pre<chapter name="chp1">post\n'
             'row 2\n'
@@ -59,7 +59,7 @@ class TestGetChapters(unittest.TestCase):
         self.assertEqual(self.page.get_chapters(), expected)
 
     def test_get_chapters_missing_attribute(self):
-        self.page.ocr = (
+        self.page.text = (
             'row 1\n'
             'pre<chapter>post\n'
             'row 2\n'
@@ -79,10 +79,10 @@ class TestRenameChapter(unittest.TestCase):
             'row 2\n'
             'row3'
         )
-        self.page = Page('0001', ocr=text)
+        self.page = Page('0001', text=text)
 
     def test_rename_chapter_no_chapter(self):
-        self.page.ocr = (
+        self.page.text = (
             'row 1\n'
             'row 2\n'
             'row3'
@@ -93,10 +93,10 @@ class TestRenameChapter(unittest.TestCase):
             'row3'
         )
         self.page.rename_chapter('chp1', 'foo')
-        self.assertEqual(self.page.ocr, expected)
+        self.assertEqual(self.page.text, expected)
 
     def test_rename_chapter_single_match(self):
-        self.page.ocr = (
+        self.page.text = (
             'row 1\n'
             'pre<chapter name="chp1">post\n'
             'row 2\n'
@@ -111,10 +111,10 @@ class TestRenameChapter(unittest.TestCase):
             'row3'
         )
         self.page.rename_chapter('chp2', 'foo')
-        self.assertEqual(self.page.ocr, expected)
+        self.assertEqual(self.page.text, expected)
 
     def test_rename_chapter_multiple_matches(self):
-        self.page.ocr = (
+        self.page.text = (
             'row 1\n'
             'pre<chapter name="chp1">post\n'
             'row 2\n'
@@ -131,7 +131,7 @@ class TestRenameChapter(unittest.TestCase):
             'pre<chapter name="chp1">post\n'
         )
         self.page.rename_chapter('chp1', 'foo')
-        self.assertEqual(self.page.ocr, expected)
+        self.assertEqual(self.page.text, expected)
 
 
 class TestCheckBlank(unittest.TestCase):
@@ -144,23 +144,23 @@ class TestCheckBlank(unittest.TestCase):
             'row 2\n'
             'row3'
         )
-        self.page = Page('0001', ocr=text)
+        self.page = Page('0001', text=text)
 
     @unittest.expectedFailure  # hack to assert not Warns
     def test_check_blank_empty_and_blank(self):
-        self.page.ocr = ''
+        self.page.text = ''
         self.page.is_proofread = None
         with self.assertWarnsRegex(UserWarning, r'is blank'):
             self.page.check_blank()
 
     def test_check_blank_empty_and_proofread(self):
-        self.page.ocr = ''
+        self.page.text = ''
         self.page.is_proofread = True
         with self.assertWarnsRegex(UserWarning, r'is blank'):
             self.page.check_blank()
 
     def test_check_blank_empty_and_not_proofread(self):
-        self.page.ocr = ''
+        self.page.text = ''
         self.page.is_proofread = False
         with self.assertWarnsRegex(UserWarning, r'might be blank'):
             self.page.check_blank()
diff --git a/tests/test_page_range.py b/tests/test_page_range.py
index f991af1..4e77bc4 100644
--- a/tests/test_page_range.py
+++ b/tests/test_page_range.py
@@ -2,6 +2,7 @@
 """Unit tests for page_range."""
 import unittest
 
+from runeberg.page import Page
 from runeberg.page_range import PageRange
 
 
@@ -113,3 +114,19 @@ class TestLast(BaseTest):
 
     def test_last(self):
         self.assertEqual(self.pages.last(), 1)
+
+
+class TestText(unittest.TestCase):
+
+    """Test the text property."""
+
+    def test_text_empty(self):
+        pages = PageRange([])
+        self.assertEqual(pages.text, '')
+
+    def test_text(self):
+        pages = PageRange([
+            ('one', Page('page_1', text='text_1')),
+            ('two', Page('page_2', text='text_2')),
+            ('three', Page('page_3', text='text_3'))])
+        self.assertEqual(pages.text, 'text_1\ntext_2\ntext_3')
diff --git a/tests/test_work.py b/tests/test_work.py
index 93148c0..a3f14b0 100644
--- a/tests/test_work.py
+++ b/tests/test_work.py
@@ -193,7 +193,7 @@ class TestLoadArticle(unittest.TestCase):
 
         patcher = mock.patch('runeberg.work.Work.parse_range')
         self.mock_parse_range = patcher.start()
-        self.mock_parse_range.return_value = [Page('0001', ocr='abc')]
+        self.mock_parse_range.return_value = [Page('0001', text='abc')]
         self.addCleanup(patcher.stop)
 
         patcher = mock.patch('runeberg.page.Page.rename_chapter')
@@ -321,9 +321,9 @@ class TestLoadArticle(unittest.TestCase):
         self.mock_disambiguation_counter.return_value = Counter({'A': 1})
         self.mock_get_chapters.return_value = ['A']
         self.mock_parse_range.return_value = [
-            Page('0001', ocr='abc'),
-            Page('0002', ocr='abc'),
-            Page('0003', ocr='abc')]
+            Page('0001', text='abc'),
+            Page('0002', text='abc'),
+            Page('0003', text='abc')]
 
         with self.assertRaises(ReconciliationError):
             self.work.load_articles(self.base_path, chapters,
@@ -418,3 +418,20 @@ class TestDetermineImageFileType(unittest.TestCase):
 
         self.work.determine_image_file_type('base', 'img')
         self.assertEquals(self.work.image_type, '.jpg')
+
+
+class TestText(unittest.TestCase):
+
+    """Unit tests for text property."""
+
+    def setUp(self):
+        self.work = Work('uid')
+
+    def test_text_no_pages(self):
+        self.assertEqual(self.work.text, '')
+
+    def test_text(self):
+        with mock.patch('runeberg.page_range.PageRange.text',
+                        new_callable=mock.PropertyMock) as mock_pagerange_text:
+            mock_pagerange_text.return_value = 'text in the page range'
+            self.assertEqual(self.work.text, 'text in the page range')