Repository: dipu-bd/lightnovel-crawler (language: python)

Issue:
Version 2.24.1



Discussion / hints:

Fix patch:
diff --git a/README.md b/README.md
index ee18e555b..b1ef479bf 100644
--- a/README.md
+++ b/README.md
@@ -362,6 +362,7 @@ You are very welcome to contribute in this project. You can:
 | https://pery.info/                           |     ✔      |           |                       |
 | https://ranobelib.me                         |            |           |                       |
 | https://readwebnovels.net                    |     ✔      |           |                       |
+| https://readlightnovels.net                  |     ✔      |           |    @PreownedFIN       |
 | https://reincarnationpalace.com              |            |           |                       |
 | https://rewayat.club                         |            |           |                       |
 | https://shalvationtranslations.wordpress.com |            |           |                       |
diff --git a/lncrawl/sources/readlightnovelsnet.py b/lncrawl/sources/readlightnovelsnet.py
new file mode 100644
index 000000000..063643f6b
--- /dev/null
+++ b/lncrawl/sources/readlightnovelsnet.py
@@ -0,0 +1,159 @@
+# -*- coding: utf-8 -*-
+import logging
+import re
+from concurrent import futures
+from ..utils.crawler import Crawler
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+search_url = 'https://readlightnovels.net/?s=%s'
+
+
+class ReadLightNovelsNet(Crawler):
+    base_url = 'https://readlightnovels.net/'
+
+    def search_novel(self, query):
+        query = query.lower().replace(' ', '+')
+        soup = self.get_soup(search_url % query)
+
+        results = []
+        for tab in soup.select('.home-truyendecu'):
+            search_title = tab.select_one('a')
+            latest = tab.select_one('.caption .label-primary').text.strip()
+            results.append({
+                'title': search_title['title'].strip(),
+                'url': self.absolute_url(
+                    tab.select_one('a')['href']),
+                'info': 'Latest chapter: %s' % (latest)
+            })
+        # end for
+
+        return results
+    # end def
+
+    def read_novel_info(self):
+        '''Get novel title, autor, cover etc'''
+        logger.debug('Visiting %s', self.novel_url)
+        soup = self.get_soup(self.novel_url)
+
+        self.novel_title = ' '.join([
+            str(x)
+            for x in soup.select_one('.title').contents
+            if not x.name
+        ]).strip()
+        logger.info('Novel title: %s', self.novel_title)
+
+        self.novel_cover = self.absolute_url(
+            soup.select_one('.book img')['src'])
+        logger.info('Novel cover: %s', self.novel_cover)
+
+        author = soup.select_one('.info').find_all('a')
+        if len(author) == 2:
+            self.novel_author = author[0].text + ' (' + author[1].text + ')'
+        else:
+            self.novel_author = author[0].text
+        logger.info('Novel author: %s', self.novel_author)
+
+        novel_id = soup.select_one('#id_post')['value']
+        logger.info('Novel id: %s', novel_id)
+
+        # This is copied from the Novelfull pagination 'hanlder' with minor tweaks
+        pagination_links = soup.select('.pagination li a')
+        pagination_page_numbers = []
+        for pagination_link in pagination_links:
+            pagination_page_numbers.append(int(pagination_link['data-page']))
+
+        page_count = max(
+            pagination_page_numbers) if pagination_page_numbers else 0
+        logger.info('Chapter list pages: %d' % page_count)
+
+        logger.info('Getting chapters...')
+        futures_to_check = {
+            self.executor.submit(
+                self.download_chapter_list,
+                i + 1,
+                novel_id
+            ): str(i)
+            for i in range(page_count + 1)
+        }
+        [x.result() for x in futures.as_completed(futures_to_check)]
+
+        # Didn't test without this, but with pagination the chapters could be in different orders
+        logger.info('Sorting chapters...')
+        self.chapters.sort(key=lambda x: x['volume'] * 1000 + x['id'])
+
+        # Copied straight from Novelfull
+        logger.info('Adding volumes...')
+        mini = self.chapters[0]['volume']
+        maxi = self.chapters[-1]['volume']
+        for i in range(mini, maxi + 1):
+            self.volumes.append({'id': i})
+        # end for
+    # end def
+
+    def download_chapter_list(self, page, novel_id):
+        # RLNs Uses an AJAX based pagination that calls this address:
+        pagination_url = 'https://readlightnovels.net/wp-admin/admin-ajax.php'
+
+        '''Download list of chapters and volumes.'''
+        url = pagination_url.split('?')[0].strip('/')
+        # url += '?action=tw_ajax&type=pagination&id=%spage=%s' % (novel_id, page)
+        soup = BeautifulSoup(
+            self.submit_form(
+                url,
+                {'action': 'tw_ajax',
+                 'type': 'pagination',
+                 'id': novel_id,
+                 'page': page}
+            ).json()['list_chap'],
+            'lxml'
+        )
+        
+        if not soup.find('body'):
+            raise ConnectionError('HTML document was not loaded properly')
+        # end if
+
+
+        for a in soup.select('ul.list-chapter li a'):
+            title = a.text.strip()
+
+            chapter_id = len(self.chapters) + 1
+            match = re.findall(r'ch(apter)? (\d+)', title, re.IGNORECASE)
+            if len(match) == 1:
+                chapter_id = int(match[0][1])
+            # end if
+
+            volume_id = 1 + (chapter_id - 1) // 100
+            match = re.findall(r'(book|vol|volume) (\d+)',
+                               title, re.IGNORECASE)
+            if len(match) == 1:
+                volume_id = int(match[0][1])
+            # end if
+
+            data = {
+                'title': title,
+                'id': chapter_id,
+                'volume': volume_id,
+                'url': self.absolute_url(a['href']),
+            }
+            self.chapters.append(data)
+        # end for
+    # end def
+
+    def download_chapter_body(self, chapter):        #
+        # NOTE: Set `chapter['body_lock'] = True` to disable post-formatting.
+        #       It can be useful in non-english sources, e.g. aixdzs, qidiancom, tiknovel
+        #
+        #       Return an empty body if anything goes wrong. But you should not return `None`.
+        '''Download body of a single chapter and return as clean html format.'''
+        logger.info('Downloading %s', chapter['url'])
+        soup = self.get_soup(chapter['url'])
+
+        contents = soup.select_one('.chapter-content')
+        for br in contents.select('br'):
+            br.decompose()
+        # end for
+
+        return str(contents)
+    # end def
+# end class


Test patch:
diff --git a/lncrawl/bots/test/test_inputs.py b/lncrawl/bots/test/test_inputs.py
index 0c521337f..6089ad252 100644
--- a/lncrawl/bots/test/test_inputs.py
+++ b/lncrawl/bots/test/test_inputs.py
@@ -315,4 +315,7 @@
     'https://foxaholic.com/': [
         'https://foxaholic.com/novel/the-white-eyed-wolves-i-personally-raised-are-all-coveting-my-legacy/'
     ],
+    'https://readlightnovels.net/': [
+        'https://readlightnovels.net/death-march-kara-hajimaru-isekai-kyusoukyoku.html'
+    ]
 }