latest update to scholarscraper:

# ------------------------------
# 
# scholar_scraper.py
import requests
from selenium import webdriver
from selenium_stealth import stealth
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selectolax.lexbor import LexborHTMLParser
from concurrent.futures import ThreadPoolExecutor, as_completed
import concurrent.futures
from typing import List, Dict

from google_scholar_py import CustomGoogleScholarOrganic
from scraper.proxy_handler import setup_proxy 
from scraper.rate_limit_manager import call_with_rate_limit
from scraper.proxy_rotation_manager import proxy_generator

import time
import random
from termcolor import colored
from typing import List, Dict, Union


class ScholarScraper:
    """Class to scrape GScholar search result and other links."""
    
    def __init__(self, max_results: int = 5, ref_search: bool = False):
        self.max_results = 1 if ref_search else max_results
        self.scholar_scrape_result: List[Dict] = []
        self.ref_search = ref_search

    def scrape_scholar_search_result(self, searchquery) -> Union[List[Dict], bool]:
        '''Scrapes Google Scholar results based on the search query.
        Returned keys in dict are: 'title', 'title_link', 'publication_info', 'snippet', 
        'cited_by_link', 'cited_by_count', 'pdf_file'.'''

        print(colored(f"Starting to scrape Google Scholar for query: {searchquery}", 'yellow'))
        
        parser = CustomGoogleScholarOrganic()

        try:
            all_results = call_with_rate_limit(
                parser.scrape_google_scholar_organic_results,
                query=searchquery,
                pagination=False,
                save_to_csv=False,
                save_to_json=False
            )

            self.scholar_scrape_result = all_results[:self.max_results]

            if self.ref_search:  
                result = self.scholar_scrape_result[0]
                print(f"Result: \n{result}\n\n")                
                # Extract authors and title from the reference line
                ref_line = searchquery
                parts = ref_line.split('.')
                authors = parts[0].strip()  # Assuming first part is the author
                title = parts[1].strip() if len(parts) > 1 else ""

                print(f"Ref_line: {ref_line} | Parts: {parts} | Authors: {authors} | Title: {title} | ")                


                # Validate against the scraped result
                if authors not in result.get('publication_info'):
                    print(colored("Author name not found in the publication info.", 'red'))
                    return False

                #if title not in result.get('title'):
                    #print(colored("Title not found in the result title.", 'red'))
                    #return False

                print(colored("Reference line validated successfully!", 'green'))
                return True

            print(colored("Successfully scraped Google Scholar results.", 'green'))

        except Exception as e:
            print(colored(f"Error while scraping Google Scholar results: {e}", 'red'))

        return self.scholar_scrape_result


    def scrape_scholar_search_result_async(self, searchquery_list):

        try:
            # Now search each returned result concurrently
            results_list = []

            # Use ThreadPoolExecutor to scrape links concurrently
            with concurrent.futures.ThreadPoolExecutor() as executor:
                parser = CustomGoogleScholarOrganic()
                futures = {executor.submit(self.scrape_scholar_search_result, searchquery): searchquery for searchquery in searchquery_list}

                for future in concurrent.futures.as_completed(futures):
                    searchquery = futures[future]
                    try:
                        data = future.result() # Single data returned by single method
                        data = data [:self.max_results]  # Limit search result based on `max_results`
                        results_list.append(data)
                    except Exception as e:
                        print(colored(f"Error while searching gscholar {searchquery}: {e}", 'red'))

                return results_list
            
        except Exception as e:
            print(colored(f"Error while scraping Google Scholar results: {e}", 'red'))
            return []

    def parse_abstract(self, parser_containing_link: LexborHTMLParser):
        '''
        This function parses the abstract from a given page and appends it to the results list.
        It prioritizes div tags over meta tags and ensures the abstract is not too short.
        '''

        possible_tags = ['div', 'p', 'span', 'h1', 'h2', 'h3', 'h4', 'li', 'td', 'tr', 'section', 'article', 'a']  # Exclude meta and specific selectors
        abstract = None

        try:
            print(colored("Starting to parse the abstract...", 'blue'))

            # First, check for the specific meta tag using CSS selector
            meta_elements = parser_containing_link.css('meta[name="eprints.abstract"]')
            for meta in meta_elements:
                content = meta.attrs.get('content', None)
                if content:
                    abstract = content
                    print(f"ABSTRACT FROM META:\n\n{abstract}")
                    print(colored("Found an abstract in <meta> tag with name='eprints.abstract'.", 'yellow'))
                    if len(abstract.split()) <= 15:
                        print(colored("Abstract is too short (<= 15 words).", 'red'))
                        abstract = None
                    else:
                        print(colored("Valid abstract found: ", 'light_magenta') + abstract)
                        return abstract  # Return immediately if found

            # If not found in meta, check other common tags
            for tag in possible_tags:
                print(colored(f"Checking for 'abstract' in <{tag}> tags...", 'cyan'))
                elements = parser_containing_link.css(tag)
                for element in elements:
                    class_attr = element.attrs.get('class', '')
                    id_attr = element.attrs.get('id', '')
                    name_attr = element.attrs.get('name', '')
                    content_attr = element.attrs.get('content', '')

                    # Abstract Identifier Logic
                    is_abstract_identified = 'abstract' in class_attr.lower() or \
                                            'abstract' in id_attr.lower() or \
                                            'abstract' in name_attr.lower() or \
                                            'abstract' in content_attr.lower()

                    if is_abstract_identified:
                        abstract_text = element.text()
                        print(f"ELEMENT:\n\n{element}")
                        print(f"ABSTRACT:\n\n{abstract_text}")
                        print(colored(f"Found an abstract in <{tag}> with class/id/name containing 'abstract'.", 'yellow'))
                        if len(abstract_text.split()) <= 15:
                            print(colored("Abstract is too short (<= 15 words).", 'red'))
                            abstract = None
                        else:
                            abstract = abstract_text
                            print(colored("Valid abstract found: ", 'light_magenta') + abstract)
                            return abstract  # Return immediately if found

            # If there's still no abstract found
            print(colored("No abstract found in <meta> tags or common tags.", 'red'))
            return "No abstract found."

        except Exception as e:
            print(colored(f"Error parsing abstract: {e}", 'red'))
            return "No abstract found."  # Return if an error occurs

    def scrape_abstract(self, url: str) -> str:
        '''
        Extracts abstracts from the provided URL.
        
        Arguments:
        - url: URL to scrape abstracts from.        
        '''

        # Check if the link leads to a PDF by making a request
        try:
            response = requests.get(url, stream=True, timeout=10)
            content_type = response.headers.get('content-type', '')            
            if 'application/pdf' in content_type.lower():
                return "This link is to a PDF document."
        except Exception as e:
            print(colored(f"Error checking URL for PDF: {e}", 'red'))

        # Check for '.pdf' or 'books.google' in the provided link
        if '.pdf' in url:
            return "This link don't return a webpage, but a PDF document."
        if 'books.google' in url:
            return "This link don't return a webpage, but a Google Books."

        options = webdriver.ChromeOptions()
        options.add_argument('--headless')
        options.add_argument('--no-sandbox')
        options.add_argument('--disable-dev-shm-usage')
        
        # Chrome options for stealth
        options.add_experimental_option('excludeSwitches', ['enable-automation', 'enable-logging'])
        options.add_experimental_option('useAutomationExtension', False) 
        
        service = Service(ChromeDriverManager().install())
        driver = webdriver.Chrome(service=service, options=options)

        stealth(driver,
            languages=['en-US', 'en'],
            vendor='Google Inc.',
            platform='Win32',
            webgl_vendor='Intel Inc.',
            renderer='Intel Iris OpenGL Engine',
            fix_hairline=True,
        )
        
        results_data = []
        
        print(colored(f"Fetching URL: {url}...", 'yellow'))
        driver.get(url)
        time.sleep(random.randint(1, 3))  # Sleep to avoid getting blocked

        parser = LexborHTMLParser(driver.page_source)
        scraped_abstract = self.parse_abstract(parser_containing_link=parser)

        driver.quit()
        print(colored("Driver closed. Scraping completed.", 'green'))

        return scraped_abstract

    # Method to extract abstract from many URLs content        
    def scrape_many_abstracts(self, urls: List) -> List:

        abstracts = []
        
        # Limit the number of threads to a maximum of 8
        with ThreadPoolExecutor(max_workers=8) as executor:
            # Submit all the scrape_abstract tasks
            future_to_url = {executor.submit(self.scrape_abstract, url): url for url in urls}
            
            # Process completed futures as they finish
            for future in as_completed(future_to_url):
                url = future_to_url[future]
                try:
                    abstract = future.result()
                    abstracts.append(abstract)
                except Exception as e:
                    print(f"Error scraping {url}: {e}")  # Handle errors gracefully

        return abstracts

# Example usage
if __name__ == "__main__":
    scraper = ScholarScraper(searchquery="sadness psychology", max_results=5)
    results = scraper.scrape_scholar_search_result()
    
    # For demonstration, print the results
    print("Scholar Results:")
    for result in results:
        print(result)