Repository: kevinzg/facebook-scraper

Issue:
Can we add a feature to take the text of the comments of each post?

Discussion:

Patch:
diff --git a/facebook_scraper/extractors.py b/facebook_scraper/extractors.py
index ff3f385..d3cec0d 100644
--- a/facebook_scraper/extractors.py
+++ b/facebook_scraper/extractors.py
@@ -95,6 +95,7 @@ class PostExtractor:
             'shared_username': None,
             'shared_post_url': None,
             'available': None,
+            'comments_full': None,
         }
 
     def extract_post(self) -> Post:
@@ -154,6 +155,13 @@ class PostExtractor:
             else:
                 post.update(reactions)
 
+        if self.options.get('comments'):
+            try:
+                comments = self.extract_comments_full()
+                post.update(comments)
+            except Exception as ex:
+                log_warning("Exception while extracting comments: %r", ex)
+
         return post
 
     def extract_post_id(self) -> PartialPost:
@@ -177,8 +185,6 @@ class PostExtractor:
             if match:
                 url = utils.urljoin(FB_MOBILE_BASE_URL, match.groups()[0].replace("&amp;", "&"))
                 response = self.request(url)
-                # If .story_body_container is commented the find function won't work
-                response.html.html = response.html.html.replace('<!--', '').replace('-->', '')
                 element = response.html.find('.story_body_container', first=True)
 
         nodes = element.find('p, header, span[role=presentation]')
@@ -507,6 +513,60 @@ class PostExtractor:
             'available': ">This content isn't available at the moment<" not in self.element.html
         }
 
+    def extract_comments_full(self):
+        """Fetch comments for an existing post obtained by `get_posts`.
+        Note that this method may raise multiple http requests per post to get all comments"""
+        url = self.post.get('post_url').replace(FB_BASE_URL, FB_MOBILE_BASE_URL)
+        logger.debug(f"Fetching {url}")
+        response = self.request(url)
+        comments = list(response.html.find('div[data-sigil="comment"]'))
+        more = response.html.find("a", containing="View more comments", first=True)
+        while more:
+            url = utils.urljoin(FB_MOBILE_BASE_URL, more.attrs.get("href"))
+            logger.debug(f"Fetching {url}")
+            response = self.request(url)
+            more_comments = response.html.find('div[data-sigil="comment"]')
+            comments.extend(more_comments)
+            more = response.html.find("a", containing="View more comments", first=True)
+        logger.debug(f"Found {len(comments)} comments")
+        result = []
+        for comment in comments:
+            comment_id = comment.attrs.get("id")
+            first_link = comment.find("div:not([data-sigil])>a[href]:not([data-click]):not([data-store]):not([data-sigil]):not([class])", first=True)
+            comment_body_elem = comment.find('[data-sigil="comment-body"]', first=True)
+            commenter_meta = None
+            if first_link:
+                url = utils.urljoin(FB_BASE_URL, first_link.attrs.get("href"))
+                name = first_link.text
+                if "\n" in name:
+                    commenter_meta, name = name.split("\n")
+            else:
+                # Adjacent div to comment body, if not logged in. No link to user page available in that case
+                url = None
+                elem = comment_body_elem.element.getprevious()
+                if elem is not None:
+                    name = elem.text
+                else:
+                    name = None
+
+            text = comment_body_elem.text
+            # Try to extract from the abbr element
+            date_element = comment.find('abbr', first=True)
+            if date_element:
+                date = utils.parse_datetime(date_element.text, search=False)
+            else:
+                date = None
+
+            result.append({
+                "comment_id": comment_id,
+                "commenter_url": url,
+                "commenter_name": name,
+                "commenter_meta": commenter_meta,
+                "comment_text": text,
+                "comment_time": date,
+            })
+        return {"comments_full": result}
+
     def parse_share_and_reactions(self, html: str):
         bad_jsons = self.shares_and_reactions_regex.findall(html)
         for bad_json in bad_jsons:
diff --git a/facebook_scraper/facebook_scraper.py b/facebook_scraper/facebook_scraper.py
index a9497ad..0cd122e 100644
--- a/facebook_scraper/facebook_scraper.py
+++ b/facebook_scraper/facebook_scraper.py
@@ -54,7 +54,6 @@ class FacebookScraper:
                 post_url = utils.urljoin(FB_MOBILE_BASE_URL, post_url)
             logger.debug(f"Requesting page from: {post_url}")
             response = self.get(post_url)
-            response.html.html = response.html.html.replace('<!--', '').replace('-->', '')
             elem = response.html.find('article[data-ft],div.async_like[data-ft]', first=True)
             post = extract_post(elem, request_fn=self.get, options=options)
             if remove_source:
@@ -68,6 +67,7 @@ class FacebookScraper:
     def get(self, url, **kwargs):
         try:
             response = self.session.get(url=url, **self.requests_kwargs, **kwargs)
+            response.html.html = response.html.html.replace('<!--', '').replace('-->', '')
             response.raise_for_status()
             return response
         except RequestException as ex:
diff --git a/facebook_scraper/utils.py b/facebook_scraper/utils.py
index dc5ae98..b7971a8 100644
--- a/facebook_scraper/utils.py
+++ b/facebook_scraper/utils.py
@@ -60,9 +60,19 @@ month = (
     r"Nov(?:ember)?|"
     r"Dec(?:ember)?"
 )
+day = (
+    r"Mon|"
+    r"Tue|"
+    r"Wed|"
+    r"Thu|"
+    r"Fri|"
+    r"Sat|"
+    r"Sun"
+)
 day_of_month = r"\d{1,2}"
 specific_date_md = f"(?:{month}) {day_of_month}" + r"(?:,? \d{4})?"
 specific_date_dm = f"{day_of_month} (?:{month})" + r"(?:,? \d{4})?"
+day_of_week = f"on ({day})"
 
 date = f"{specific_date_md}|{specific_date_dm}|Today|Yesterday"
 
@@ -75,7 +85,7 @@ relative_time_hours = r"\b\d{1,2} ?h(?:rs?)?"
 relative_time_mins = r"\b\d{1,2} ?mins?"
 relative_time = f"{relative_time_hours}|{relative_time_mins}"
 
-datetime_regex = re.compile(fr"({exact_time}|{relative_time})", re.IGNORECASE)
+datetime_regex = re.compile(fr"({exact_time}|{relative_time}|{day_of_week})", re.IGNORECASE)
 
 
 def parse_datetime(text: str, search=True) -> Optional[datetime]:


Test patch:
diff --git a/tests/test_parse_date.py b/tests/test_parse_date.py
index 315a3e0..2035cc5 100644
--- a/tests/test_parse_date.py
+++ b/tests/test_parse_date.py
@@ -31,6 +31,7 @@ class TestParseDate:
         '16 hrs',
         '1 min',
         '50 mins',
+        'on Sat',
     ]
 
     def test_all_dates(self):