Repository: lokal-profil/runeberg

Issue:
Works with .jpg images
Of course not all scanned images are .tif. See e.g. [`aoshbok`](runeberg.org/aoshbok/)

When not these will crash work parsing as `page.from_path()` assumed `.tif`.

DjVu conversion also assumes `.tif` (this is expected by `cjb2`).

Determine the file type and store this as a variable in work which is passed on to `page.from_path()` and `work.can_djvu()`.

Comments:

Patch:
diff --git a/runeberg/page.py b/runeberg/page.py
index a4f435b..604c251 100644
--- a/runeberg/page.py
+++ b/runeberg/page.py
@@ -39,6 +39,13 @@ class Page(object):
         """Represent the Page as a string."""
         return self.uid
 
+    @property
+    def image_file_type(self):
+        """Return the file extension of the image file."""
+        if not self.image:
+            return None
+        return os.path.splitext(self.image)[1]
+
     def set_blank(self, default_label=None):
         """
         Set the blank proofread status based on page label.
@@ -74,7 +81,7 @@ class Page(object):
                 UserWarning)
 
     @staticmethod
-    def from_path(base_path, uid, label, whole_page_ok):
+    def from_path(base_path, uid, image_type, label, whole_page_ok):
         """
         Create a page from the path to the unzipped files and a uid.
 
@@ -82,11 +89,13 @@ class Page(object):
             unzipped files.
         @param uid: the identifier of the page, ordinarily the zero-padded
             ordinal number of the page.
+        @param image_type: the expected file extension of the scanned image.
         @param label: the real printable label of the page, ordinarily the page
             number.
         @param whole_page_ok: whether the whole page has been proofread
         """
-        img_path = os.path.join(base_path, IMG_DIR, '{}.tif'.format(uid))
+        img_path = os.path.join(
+            base_path, IMG_DIR, '{0}{1}'.format(uid, image_type))
         if not os.path.isfile(img_path):
             raise ValueError(
                 '{} was provided as the image for "{}" but no such image '
diff --git a/runeberg/work.py b/runeberg/work.py
index 1aad793..9cefb1f 100644
--- a/runeberg/work.py
+++ b/runeberg/work.py
@@ -11,18 +11,21 @@ This most often corresponds to an edition in e.g. Wikidata terminology.
 import os
 from collections import Counter, OrderedDict
 from itertools import takewhile
+from shlex import quote
 from shutil import which  # used for djvu conversion
 from subprocess import DEVNULL, run  # used for djvu conversion
 
 from tqdm import tqdm
 
 from runeberg.article import Article
-from runeberg.download import DATA_DIR, SITE, UNZIP_SUBDIR
+from runeberg.download import DATA_DIR, IMG_DIR, SITE, UNZIP_SUBDIR
 from runeberg.lst_file import LstFile
 from runeberg.page import Page
 from runeberg.page_range import PageRange
 from runeberg.person import Person
 
+IMAGE_TYPES = ('.tif', '.jpg')
+
 
 class Work(object):
     """An object representing a runeberg.org work."""
@@ -41,11 +44,12 @@ class Work(object):
         self.identifiers = {}  # identifiers for the work in external sources
         self.people = {}  # people involved with the work
         self.djvu = None  # djvu file of the whole work
+        self.image_type = None  # the default file extension of the image files
 
         # @TODO: something recording the date when runeberg was downloaded
 
     @staticmethod
-    def from_files(uid, base_path=None, known_people=None):
+    def from_files(uid, base_path=None, known_people=None, img_dir=None):
         """
         Create an Work from the downloaded, unzipped files.
 
@@ -53,10 +57,14 @@ class Work(object):
             Overrides the default {cwd}/DATA_DIR/{uid}/UNZIP_SUBDIR
         @param known_people: dict of Person objects from which author, editor
             and translator roles are matched.
+        @param img_dir: the name given to the image sub-directory. Defaults to
+            IMG_DIR.
         """
         work = Work(uid)
         base_path = base_path or os.path.join(
             os.getcwd(), DATA_DIR, uid, UNZIP_SUBDIR)
+        img_dir = img_dir or IMG_DIR
+        work.determine_image_file_type(base_path, img_dir)
         chapters = work.load_pages(base_path)
         work.load_articles(base_path, chapters)
         work.load_metadata(base_path)
@@ -68,6 +76,32 @@ class Work(object):
         """Return the base url on runeberg.org."""
         return '{0}/{1}/'.format(SITE, self.uid)
 
+    def determine_image_file_type(self, base_path, img_dir):
+        """
+        Determine the file type of the images of the scanned pages.
+
+        Will stop after encountering the first recognised image file type.
+
+        @raises NoImagesError: if no files are encountered
+        @raises UnrecognisedImageTypeError: if no recognised image file types
+            are encountered.
+        @param base_path: The path to the unzipped directory.
+        @param img_dir: the image subdirectory
+        """
+        found_types = set()
+        img_dir_path = os.path.join(base_path, img_dir)
+        with os.scandir(img_dir_path) as it:
+            for entry in it:
+                if not entry.name.startswith('.') and entry.is_file():
+                    ext = os.path.splitext(entry)[1]
+                    if ext in IMAGE_TYPES:
+                        self.image_type = ext
+                        return
+                    found_types.add(ext)
+        if not found_types:
+            raise NoImagesError(img_dir_path)
+        raise UnrecognisedImageTypeError(found_types)
+
     def load_pages(self, base_path):
         """
         Populate self.pages.
@@ -86,7 +120,7 @@ class Work(object):
         chapters = Counter()
         for uid, label in pages.data:
             page = Page.from_path(
-                base_path, uid, label, uid in ok_pages)
+                base_path, uid, self.image_type, label, uid in ok_pages)
             chapters.update(page.get_chapters())
             self.pages[uid] = page
         return chapters
@@ -282,10 +316,22 @@ class Work(object):
         stderr = DEVNULL if silent else None
         for i, page in tqdm(enumerate(self.pages.values(), 1),
                             total=len(self.pages)):
-            # cjb2 to convert single tif to djvu
+            in_file = quote(page.image)
+
             # @TODO: can try "capture_output=True" if python is bumped to 3.7
-            run(['cjb2', '-clean', page.image, tmp_djvu],
-                check=True, stderr=stderr)
+            # use page file type in case original image has been replaced
+            if page.image_file_type == '.tif':
+                # cjb2 to convert single tif to djvu
+                run(['cjb2', '-clean', in_file, tmp_djvu],
+                    check=True, stderr=stderr)
+            elif page.image_file_type == '.jpg':
+                run(['c44', '-crcbfull', in_file, tmp_djvu],
+                    check=True, stderr=stderr)
+            else:
+                raise NotImplementedError(
+                    'At the moment only .tif and .jpg images can be converted '
+                    'to DjVu')
+
             if i == 1:
                 base_path = os.path.dirname(page.image)
                 book_djvu = os.path.join(base_path, 'book.djvu')
@@ -315,6 +361,33 @@ class Work(object):
         return (which('djvm') is not None and which('cjb2') is not None)
 
 
+class NoImagesError(Exception):
+    """Error when no images were found for the pages of the work."""
+
+    def __init__(self, img_dir_path):
+        """Initialise a NoImagesError."""
+        msg = (
+            'Could not detect any images for this work. Are you sure you '
+            'provided the correct image subdirectory ({}) and that this is '
+            'not a volumed work?'.format(img_dir_path))
+        super().__init__(msg)
+
+
+class UnrecognisedImageTypeError(Exception):
+    """Error in trying to determine the image file type."""
+
+    def __init__(self, encountered):
+        """Initialise a UnrecognisedImageTypeError."""
+        encountered = sorted(list(encountered))  # make deterministic
+        msg = (
+            'The encountered images do not seem to be of a recognised image '
+            'file type.\n'
+            'Recognised image file types are: {0}\n'
+            'Encountered file types: {1}'.format(
+                ', '.join(IMAGE_TYPES), ', '.join(encountered)))
+        super().__init__(msg)
+
+
 class DisambiguationError(Exception):
     """Error in disambiguating non-unique article titles."""
 


Test patch:
diff --git a/tests/test_page.py b/tests/test_page.py
index c0a84e7..94f8c39 100644
--- a/tests/test_page.py
+++ b/tests/test_page.py
@@ -183,3 +183,19 @@ class TestCheckBlank(unittest.TestCase):
         self.page.is_proofread = False
         with self.assertWarnsRegex(UserWarning, r'is not empty'):
             self.page.check_blank()
+
+
+class TestImageFileType(unittest.TestCase):
+
+    """Test the image_file_type() method."""
+
+    def setUp(self):
+        self.page = Page('0001')
+
+    def test_image_file_type_empty(self):
+        # self.page.image = ''
+        self.assertIsNone(self.page.image_file_type)
+
+    def test_image_file_type_tif(self):
+        self.page.image = 'somewhere/foo.tif'
+        self.assertEquals(self.page.image_file_type, '.tif')
diff --git a/tests/test_work.py b/tests/test_work.py
index 599e9d8..93148c0 100644
--- a/tests/test_work.py
+++ b/tests/test_work.py
@@ -7,7 +7,29 @@ from collections import Counter, OrderedDict
 from runeberg.article import Article
 from runeberg.page import Page
 from runeberg.page_range import PageRange
-from runeberg.work import DisambiguationError, ReconciliationError, Work
+from runeberg.work import (
+    DisambiguationError,
+    NoImagesError,
+    ReconciliationError,
+    UnrecognisedImageTypeError,
+    Work
+)
+
+
+class PseudoDirEntry(object):
+
+    """Class to mock DirEntry returned by os.scandir."""
+
+    def __init__(self, name, is_file=True):
+        self.name = name
+        self.path = './somewhere/{}'.format(name)
+        self._is_file = is_file
+
+    def is_file(self):
+        return self._is_file
+
+    def __fspath__(self):
+        return self.path
 
 
 class TestParseRange(unittest.TestCase):
@@ -343,3 +365,56 @@ class TestReconciliationError(unittest.TestCase):
         self.assertEqual(e.no_tag_articles, [
             'b: page(s) 002'
         ])
+
+
+class TestDetermineImageFileType(unittest.TestCase):
+
+    """Unit tests for determine_image_file_type()."""
+
+    def setUp(self):
+        self.work = Work('test')
+        # cannot autospec due to https://bugs.python.org/issue23078
+
+        self.mock_scandir_list = [
+            PseudoDirEntry('.git', False),
+            PseudoDirEntry('.travis.yml'),
+            PseudoDirEntry('tests', False)
+        ]
+
+        self.mock_scandir_val = mock.MagicMock()
+        self.mock_scandir_val.__enter__.return_value = self.mock_scandir_list
+        patcher = mock.patch('runeberg.work.os.scandir',
+                             return_value=self.mock_scandir_val)
+        self.mock_scandir = patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def test_determine_image_file_type_empty(self):
+        del self.mock_scandir_list[:]  # empty the list without creating new
+        with self.assertRaises(NoImagesError):
+            self.work.determine_image_file_type('base', 'img')
+            self.mock_scandir.assert_called_once_with('base/img')
+
+    def test_determine_image_file_type_no_valid_files(self):
+        with self.assertRaises(NoImagesError):
+            self.work.determine_image_file_type('base', 'img')
+
+    def test_determine_image_file_type_valid_file(self):
+        self.mock_scandir_list.append(PseudoDirEntry('tests.jpg'))
+
+        self.work.determine_image_file_type('base', 'img')
+        self.assertEquals(self.work.image_type, '.jpg')
+
+    def test_determine_image_file_type_no_valid_types(self):
+        self.mock_scandir_list.append(PseudoDirEntry('tests.foo'))
+        self.mock_scandir_list.append(PseudoDirEntry('tests.bar'))
+
+        with self.assertRaisesRegex(UnrecognisedImageTypeError,
+                                    '\.bar, \.foo'):
+            self.work.determine_image_file_type('base', 'img')
+
+    def test_determine_image_file_type_valid_file_ignore_second(self):
+        self.mock_scandir_list.append(PseudoDirEntry('tests.jpg'))
+        self.mock_scandir_list.append(PseudoDirEntry('tests.tif'))
+
+        self.work.determine_image_file_type('base', 'img')
+        self.assertEquals(self.work.image_type, '.jpg')