add support for using semantic scholar

crew/research_article_suggester.py

@@ -1,6 +1,7 @@
 from crewai import Agent, Task, Crew
 from langchain_openai import ChatOpenAI
 from tavily import TavilyClient
+from semanticscholar import SemanticScholar
 import arxiv
 import os
 import json
@@ -54,6 +55,21 @@ class RecentArticleSuggester:
             }
             results.append(paper)
 
+        print("\nSearching for papers on Semanticscholar...")
+        sch = SemanticScholar()
+        semantic_results = sch.search_paper(
+            self.topic, sort='publicationDate:desc', bulk=True,
+            fields=['title', 'url', 'authors', 'publicationDate', 'abstract'])
+        for result in semantic_results[:MAX_RESULTS]:
+            paper = {
+                "title": result.title,
+                "authors": ", ".join(str(author.name) for author in result.authors),
+                "content": result.abstract,
+                "published_on": result.publicationDate,
+                "url": result.url,
+            }
+            results.append(paper)
+
         # pitch_crew = self._create_pitch_crew()
         research_paper_suggestions = []
         for result in results:
@@ -86,8 +102,8 @@ class RecentArticleSuggester:
                 f"Formatting Instructions: {parser.get_format_instructions()}"
             ),
             HumanMessage(
-                f"Here is the information about the research paper title: {article['title']}, url: {article['url']},"
-                f" summary: \n{article['content']}.\n\n Research Paper content:\n{article_content}"
+                f"Here is the information about the research paper:\n {article}\n\n"
+                f"Research Paper content:\n{article_content}"
             )
         ])
         llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0.2)
@@ -127,9 +143,8 @@ class RecentArticleSuggester:
                 f"Formatting Instructions: {pitch_parser.get_format_instructions()}"
             ),
             HumanMessage(
-                f"Here is the information about the research paper title: {article_info['title']}, url: {article_info['url']}, "
-                f"published_on: {article_info['published_on']}, authors: {article_info['author']}, "
-                f"summary: \n{article_info['summary']}.\n\n Research Paper content:\n{article_info['article_content']}"
+                f"Here is the information about the research paper:\n {article_info}\n\n"
+                f"Research Paper content:\n{article_info['article_content']}"
             )
         ])
         pitch_llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0.2)

requirements.txt

@@ -7,3 +7,4 @@ langchain_openai
 streamlit
 tavily-python
 arxiv
+semanticscholar