threading implemented, Backend configured

.env

@@ -1,5 +0,0 @@
-# Gemini
-GEMINI_API_KEY=YOUR_API_KEY
-
-# MongoDB
-MONGO_URI=your_mongo_uri

.gitignore

@@ -0,0 +1,2 @@
+.env
+__pycache__/

paperflux/main.py

@@ -1,30 +1,24 @@
-import signal
-import sys
+import logging
 from src.scheduler.jobs import PaperProcessingScheduler
 from src.web.app import PaperFluxUI
-import threading
+import streamlit as st
 
-def signal_handler(signum, frame):
-    print("\nShutting down gracefully...")
-    scheduler.stop()
-    sys.exit(0)
+# logger
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("paperflux.main")
+logger.info("Initializing PaperFlux")
 
-def main():
-    global scheduler
-    
-    # Set up signal handlers
-    signal.signal(signal.SIGINT, signal_handler)
-    signal.signal(signal.SIGTERM, signal_handler)
+# Initialize scheduler
+logger.info("Creating scheduler")
+scheduler = PaperProcessingScheduler()
 
-    # Start the scheduler in a background thread
-    scheduler = PaperProcessingScheduler()
-    scheduler_thread = threading.Thread(target=scheduler.start, daemon=True)
-    scheduler_thread.start()
+# Start scheduler
+logger.info("Starting scheduler")
+scheduler.start()
+logger.info("Scheduler started")
 
-    # Create and launch the Gradio interface
-    ui = PaperFluxUI()
-    interface = ui.create_interface()
-    interface.launch(server_name="0.0.0.0", share=True)
-
-if __name__ == "__main__":
-    main()
+# Create and render UI
+logger.info("Creating UI")
+ui = PaperFluxUI(scheduler=scheduler)
+logger.info("Rendering UI")
+ui.render_app()

paperflux/src/config/settings.py

@@ -1,9 +1,9 @@
 import os
 
-MONGODB_URI = "mongodb+srv:"
+MONGODB_URI = ""
 DB_NAME = "papers_summary_database"
 COLLECTION_NAME = "papers"
 HF_API_URL = "https://huggingface.co/api/daily_papers"
 PDF_BASE_URL = "https://arxiv.org/pdf/{id}.pdf"
 TEMP_DIR = "temp_papers"
-GEMINI_API_KEY = os.getenv("GEMINI_API_KEY")
+GEMINI_API_KEY = ""

paperflux/src/scheduler/jobs.py

@@ -5,6 +5,8 @@ import os
 from src.services.paper_fetcher import PaperFetcher
 from src.services.paper_analyzer import PaperAnalyzer
 from src.services.database import DatabaseService
+import threading
+from concurrent.futures import ThreadPoolExecutor
 
 class PaperProcessingScheduler:
     def __init__(self):
@@ -13,6 +15,52 @@ class PaperProcessingScheduler:
         self.analyzer = PaperAnalyzer()
         self.db = DatabaseService()
         self._running = False
+        self.paper_processed_event = asyncio.Event()
+        self._lock = threading.Lock()
+        self.refresh_callbacks = []
+
+    def register_refresh_callback(self, callback):
+        """Register a callback to be called when a paper is processed"""
+        with self._lock:
+            self.refresh_callbacks.append(callback)
+
+    def unregister_refresh_callback(self, callback):
+        """Unregister a callback"""
+        with self._lock:
+            if callback in self.refresh_callbacks:
+                self.refresh_callbacks.remove(callback)
+
+    def _notify_refresh(self):
+        """Notify all registered callbacks that a paper has been processed"""
+        with self._lock:
+            callbacks = list(self.refresh_callbacks)
+        
+        for callback in callbacks:
+            try:
+                callback()
+            except Exception as e:
+                print(f"Error in refresh callback: {str(e)}")
+
+    def analyze_and_store_paper(self, paper_entry, pdf_path):
+        """Analyze a paper and store it in the database"""
+        try:
+            explanation = self.analyzer.analyze_paper(pdf_path)
+            paper_obj = self.fetcher.parse_paper_data(paper_entry)
+            paper_obj.explanation = explanation
+            self.db.insert_paper(paper_obj)
+            
+            self._notify_refresh()
+            
+            return True
+        except Exception as e:
+            print(f"Error analyzing paper {paper_entry['paper']['id']}: {str(e)}")
+            return False
+        finally:
+            if os.path.exists(pdf_path):
+                try:
+                    os.remove(pdf_path)
+                except:
+                    pass
 
     async def process_papers(self):
         if self._running:
@@ -24,23 +72,29 @@ class PaperProcessingScheduler:
         
         try:
             self.db.clear_collection()
+            # Fetch list of all papers
             papers = await self.fetcher.fetch_papers()
             
-            for paper in papers:
-                if not self._running:  # Check if we should stop
-                    break
-                    
-                pdf_path = await self.fetcher.download_paper(paper)
-                if pdf_path:
-                    try:
-                        explanation = self.analyzer.analyze_paper(pdf_path)
-                        paper_obj = self.fetcher.parse_paper_data(paper)
-                        paper_obj.explanation = explanation
-                        self.db.insert_paper(paper_obj)
-                    finally:
-                        if os.path.exists(pdf_path):
-                            os.remove(pdf_path)
+            # Download all papers in parallel(BG thread)
+            paper_paths = await self.fetcher.download_papers(papers)
+
+            with ThreadPoolExecutor(max_workers=2) as executor:
+                futures = []
+
+                for paper in papers:
+                    paper_id = paper["paper"]["id"]
+                    if paper_id in paper_paths:
+                        futures.append(
+                            executor.submit(
+                                self.analyze_and_store_paper,
+                                paper,
+                                paper_paths[paper_id]
+                            )
+                        )
                             
+                for future in futures:
+                    future.result()
+                    
         except Exception as e:
             print(f"Error in paper processing: {str(e)}")
         finally:

paperflux/src/services/database.py

@@ -1,21 +1,70 @@
+import time
 from pymongo import MongoClient
 from src.config.settings import MONGODB_URI, DB_NAME, COLLECTION_NAME
-from src.models.paper import Paper
+from src.models.models import Paper
+import threading
 
 class DatabaseService:
+    _instance = None
+    _lock = threading.Lock()
+
+    def __new__(cls):
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = super(DatabaseService, cls).__new__(cls)
+                cls._instance._initialized = False
+        return cls._instance
+
     def __init__(self):
+        if self._initialized:
+            return
         self.client = MongoClient(MONGODB_URI)
         self.db = self.client[DB_NAME]
         self.collection = self.db[COLLECTION_NAME]
+        self._cache = {}
+        self._cache_timestamp = 0
+        self._cache_lock = threading.Lock()
+        self._initialized = True
 
     def clear_collection(self):
         self.collection.delete_many({})
+        with self._cache_lock:
+            self._cache = {}
+            self._cache_timestamp = 0
 
     def insert_paper(self, paper: Paper):
-        return self.collection.insert_one(paper.to_dict())
+        result = self.collection.insert_one(paper.to_dict())
+        # Invalidate cache
+        with self._cache_lock:
+            self._cache = {}
+            self._cache_timestamp = 0
+        return result
+    
+    def get_all_papers(self, max_cache_age_seconds=10):
+        """Get all papers, with caching for better performance"""
+        current_time = time.time()
+
+        # check cache validity
+        with self._cache_lock:
+            if self._cache and current_time - self._cache_timestamp <= max_cache_age_seconds:
+                return self._cache.get('all_papers', [])
+            
+        # cache miss
+        papers = list(self.collection.find())
 
-    def get_all_papers(self):
-        return list(self.collection.find())
+        # update cache
+        with self._cache_lock:
+            self._cache['all_papers'] = papers
+            self._cache_timestamp = current_time
+        
+        return papers
 
     def get_paper_by_id(self, paper_id: str):
-        return self.collection.find_one({"paper_id": paper_id})
+        """Get a paper by ID with caching"""
+        with self._cache_lock:
+            if 'all_papers' in self._cache:
+                for paper in self._cache['all_papers']:
+                    if paper['id'] == paper_id:
+                        return paper
+        # cache miss
+        return self.collection.find_one({'id': paper_id})

paperflux/src/services/paper_fetcher.py

@@ -4,7 +4,7 @@ import asyncio
 from datetime import datetime
 from typing import List, Tuple, Optional
 from src.config.settings import HF_API_URL, PDF_BASE_URL, TEMP_DIR
-from src.models.paper import Paper
+from paperflux.src.models.models import Paper
 
 class PaperFetcher:
     def __init__(self):
@@ -47,43 +47,20 @@ class PaperFetcher:
             print(f"Error downloading {paper_id}: {str(e)}")
             return None
 
-    async def download_all_papers(self, papers: List[dict]) -> List[Tuple[str, bool]]:
+    async def download_papers(self, papers: List[dict]) -> List[Tuple[str, bool]]:
         """Download all papers in parallel."""
-        async with aiohttp.ClientSession() as session:
-            tasks = []
-            for paper in papers:
-                paper_id = paper["paper"]["id"]
-                pdf_url = PDF_BASE_URL.format(id=paper_id)
-                clean_id = paper_id.replace("/", "_")
-                filename = f"{datetime.now().date()}_{clean_id}.pdf"
-                filepath = os.path.join(TEMP_DIR, filename)
-
-                tasks.append(self.download_single_paper(session, paper_id, pdf_url, filepath))
-            
-            results = await asyncio.gather(*tasks)
-            successful = sum(1 for status in results if status[1])
-            print(f"Downloaded {successful}/{len(papers)} papers successfully")
-            return results
+        tasks = []
+        for paper in papers:
+            tasks.append(self.download_paper(paper))
 
-    async def download_single_paper(
-        self, 
-        session: aiohttp.ClientSession, 
-        paper_id: str, 
-        pdf_url: str, 
-        filepath: str
-    ) -> Tuple[str, bool]:
-        """Download a single paper with the given session."""
-        try:
-            async with session.get(pdf_url) as response:
-                if response.status == 200:
-                    content = await response.read()
-                    with open(filepath, "wb") as f:
-                        f.write(content)
-                    return (paper_id, True)
-                return (paper_id, False)
-        except Exception as e:
-            print(f"Error downloading {paper_id}: {str(e)}")
-            return (paper_id, False)
+        results = await asyncio.gather(*tasks)
+        paper_paths = {}
+        for paper, file_path in zip(papers, results):
+            if file_path:
+                paper_paths[paper["paper"]["id"]] = file_path
+        successful = sum(1 for status in results if status[1])
+        print(f"Downloaded {successful}/{len(papers)} papers successfully")
+        return paper_paths
 
     def parse_paper_data(self, paper_entry: dict) -> Paper:
         """Convert raw paper data to Paper model."""

paperflux/src/web/app.py

@@ -1,95 +1,237 @@
-import gradio as gr
+import streamlit as st
+import asyncio
+import threading
+import time
+import logging
+from src.scheduler.jobs import PaperProcessingScheduler
 from src.services.database import DatabaseService
 
+logger = logging.getLogger("paperflux.app")
+
 class PaperFluxUI:
-    def __init__(self):
+    def __init__(self, scheduler: PaperProcessingScheduler):
+        logger.info("Initializing PaperFluxUI")
+        self.scheduler = scheduler
         self.db = DatabaseService()
-        self.papers = self.db.get_all_papers()
-        self.current_index = 0
-
-    def get_current_paper(self):
-        if not self.papers:
-            return {
-                "title": "No papers available", 
-                "explanation": "Please wait for papers to be processed.", 
-                "pdf_url": ""
-            }
-        paper = self.papers[self.current_index]
-        authors = ", ".join([author["name"] for author in paper["authors"]])
-        title = f"# {paper['title']}\n\nAuthors: {authors}"
-        return {
-            "title": title,
-            "explanation": paper["explanation"],
-            "pdf_url": paper["pdf_url"]
+        self.callback_registered = False
+        
+        # Register callback if not already done
+        if not self.callback_registered:
+            logger.info("Registering refresh callback")
+            self.scheduler.register_refresh_callback(self.refresh_callback)
+            self.callback_registered = True
+            
+        # Set up the page configuration
+        logger.info("Setting up page config")
+        st.set_page_config(
+            page_title="PaperFlux - Research Paper Summaries",
+            page_icon="📚",
+            layout="wide",
+            initial_sidebar_state="expanded"
+        )
+        
+        # Add custom CSS
+        st.markdown("""
+        <style>
+        .main {
+            padding: 2rem;
         }
-
-    def next_paper(self):
-        if self.current_index < len(self.papers) - 1:
-            self.current_index += 1
-        return self.get_current_paper()
-
-    def previous_paper(self):
-        if self.current_index > 0:
-            self.current_index -= 1
-        return self.get_current_paper()
-
-    def create_interface(self):
-        with gr.Blocks(theme=gr.themes.Base()) as interface:
-            title = gr.Markdown()
-            explanation = gr.Markdown()
+        .paper-title {
+            font-size: 1.8rem;
+            font-weight: bold;
+            margin-bottom: 1rem;
+        }
+        .author-list {
+            margin-bottom: 1rem;
+            color: #666;
+        }
+        .summary-header {
+            font-size: 1.3rem;
+            font-weight: bold;
+            margin-top: 1rem;
+            margin-bottom: 0.5rem;
+        }
+        .explanation-header {
+            font-size: 1.5rem;
+            font-weight: bold;
+            margin-top: 2rem;
+            margin-bottom: 1rem;
+        }
+        .paper-date {
+            color: #888;
+            font-style: italic;
+        }
+        .paper-container {
+            padding: 1.5rem;
+            border-radius: 10px;
+            background-color: #f8f9fa;
+            margin-bottom: 2rem;
+        }
+        </style>
+        """, unsafe_allow_html=True)
+        logger.info("PaperFluxUI initialization complete")
+        
+    def refresh_callback(self):
+        """Callback that will be called when a new paper is processed"""
+        logger.info("Refresh callback triggered")
+        # Use Streamlit's session state to signal a refresh is needed
+        if 'needs_rerun' not in st.session_state:
+            st.session_state['needs_rerun'] = True
+    
+    def render_app(self):
+        """Render the main app content"""
+        # App header
+        col1, col2 = st.columns([5, 1])
+        with col1:
+            st.title("📚 PaperFlux")
+            st.subheader("Curated Research Papers with AI-Generated Summaries")
+        with col2:
+            if st.button("Refresh Data", key="refresh_button"):
+                # Clear session state
+                if 'current_paper_index' in st.session_state:
+                    del st.session_state['current_paper_index']
+                # Trigger rerun
+                st.rerun()
+        
+        # Get papers from database
+        papers = self.db.get_all_papers()
+        
+        # Show processing status if no papers
+        if not papers:
+            st.info("⏳ Waiting for papers to be processed. Please wait or check back later.")
             
-            # Create an HTML component for the download link
-            download_html = gr.HTML()
+            # Add progress indicator
+            if self.scheduler._running:
+                st.markdown("### 🔄 Paper processing is currently running...")
+                progress = st.progress(0)
+                for i in range(100):
+                    # Simulating progress as we don't know the actual progress
+                    time.sleep(0.1)
+                    progress.progress(i + 1)
+                    # Break if papers are available or processing stopped
+                    updated_papers = self.db.get_all_papers(max_cache_age_seconds=1)
+                    if updated_papers or not self.scheduler._running:
+                        if updated_papers:
+                            st.success("✅ Papers have been processed!")
+                            time.sleep(1)
+                            st.rerun()
+                        break
+            else:
+                st.warning("Paper processing is not currently running. It may be scheduled for midnight.")
+                
+            # Add manual trigger button
+            if st.button("Process Papers Now", key="process_now"):
+                st.info("Starting paper processing...")
+                # Use threading to avoid blocking the Streamlit interface
+                threading.Thread(
+                    target=lambda: asyncio.run(self.scheduler.process_papers()),
+                    daemon=True
+                ).start()
+                st.rerun()
+                
+            return
+        
+        # Sidebar for navigation
+        with st.sidebar:
+            st.header("Navigation")
             
-            with gr.Row():
-                prev_btn = gr.Button("Previous Paper")
-                next_btn = gr.Button("Next Paper")
-
-            def update_ui(paper_data):
-                download_link = f"""
-                <div style="text-align: center; margin-top: 10px;">
-                    <a href="{paper_data['pdf_url']}" target="_blank" 
-                       style="text-decoration: none;">
-                        <button style="padding: 10px 20px; background-color: #4CAF50; 
-                                     color: white; border: none; border-radius: 5px; 
-                                     cursor: pointer;">
-                            Download Paper
-                        </button>
-                    </a>
-                </div>
-                """
-                return (
-                    paper_data["title"],
-                    paper_data["explanation"],
-                    download_link
-                )
-
-            next_btn.click(
-                fn=lambda: update_ui(self.next_paper()),
-                outputs=[title, explanation, download_html]
+            # Store current paper index in session state
+            if 'current_paper_index' not in st.session_state:
+                st.session_state['current_paper_index'] = 0
+                
+            # Paper selection widget
+            paper_titles = [p['title'] for p in papers]
+            selected_title = st.selectbox(
+                "Select Paper",
+                paper_titles,
+                index=st.session_state['current_paper_index']
             )
             
-            prev_btn.click(
-                fn=lambda: update_ui(self.previous_paper()),
-                outputs=[title, explanation, download_html]
+            # Update current paper index when selection changes
+            st.session_state['current_paper_index'] = paper_titles.index(selected_title)
+            
+            # Navigation buttons
+            col1, col2 = st.columns(2)
+            with col1:
+                prev_disabled = st.session_state['current_paper_index'] <= 0
+                if st.button("Previous", disabled=prev_disabled):
+                    st.session_state['current_paper_index'] -= 1
+                    st.rerun()
+            
+            with col2:
+                next_disabled = st.session_state['current_paper_index'] >= len(papers) - 1
+                if st.button("Next", disabled=next_disabled):
+                    st.session_state['current_paper_index'] += 1
+                    st.rerun()
+            
+            st.markdown(f"Paper {st.session_state['current_paper_index'] + 1} of {len(papers)}")
+            
+            # Additional information
+            st.subheader("Information")
+            st.info(
+                "Papers are automatically refreshed daily at midnight. "
+                "The database is cleared and new papers are downloaded and processed."
             )
-
-            # Initialize with first paper
-            paper_data = self.get_current_paper()
-            init_download_link = f"""
-            <div style="text-align: center; margin-top: 10px;">
-                <a href="{paper_data['pdf_url']}" target="_blank" 
-                   style="text-decoration: none;">
-                    <button style="padding: 10px 20px; background-color: #4CAF50; 
-                                 color: white; border: none; border-radius: 5px; 
-                                 cursor: pointer;">
-                        Download Paper
-                    </button>
-                </a>
-            </div>
-            """
-            title.value = paper_data["title"]
-            explanation.value = paper_data["explanation"]
-            download_html.value = init_download_link
-
-        return interface
+            
+            # Show processing date
+            st.caption("Last Updated:")
+            if papers and 'processed_at' in papers[st.session_state['current_paper_index']]:
+                processed_time = papers[st.session_state['current_paper_index']]['processed_at']
+                st.caption(f"{processed_time.strftime('%Y-%m-%d %H:%M:%S')} UTC")
+        
+        # Display selected paper
+        if papers:
+            current_paper = papers[st.session_state['current_paper_index']]
+            self.display_paper(current_paper)
+    
+    def display_paper(self, paper):
+        """Display a single paper with all its details"""
+        # Paper title
+        st.markdown(f"<h1 class='paper-title'>{paper['title']}</h1>", unsafe_allow_html=True)
+        
+        # Publication date and authors
+        col1, col2 = st.columns([1, 3])
+        with col1:
+            published_date = paper.get('published_at', '')
+            if published_date:
+                try:
+                    if isinstance(published_date, str):
+                        formatted_date = published_date.split('T')[0]
+                    else:
+                        formatted_date = published_date.strftime("%Y-%m-%d")
+                    st.markdown(f"**Published:** {formatted_date}")
+                except:
+                    st.markdown(f"**Published:** {published_date}")
+        
+        with col2:
+            # Format authors
+            authors = paper.get('authors', [])
+            if authors:
+                if isinstance(authors[0], dict) and 'name' in authors[0]:
+                    author_names = [a.get('name', '') for a in authors]
+                else:
+                    author_names = [str(a) for a in authors]
+                
+                st.markdown(f"**Authors:** {', '.join(author_names)}")
+        
+        # PDF download button
+        if paper.get('pdf_url'):
+            st.markdown("### 📄 Paper Download")
+            st.markdown(f"[Download Original PDF]({paper['pdf_url']})")
+        
+        # Paper summary
+        st.markdown("<h2 class='summary-header'>Abstract</h2>", unsafe_allow_html=True)
+        st.markdown(paper.get('summary', 'No summary available.'))
+        
+        # Paper explanation
+        if paper.get('explanation'):
+            st.markdown("<h2 class='explanation-header'>AI Analysis</h2>", unsafe_allow_html=True)
+            
+            with st.expander("Show Full Analysis", expanded=True):
+                st.markdown(paper['explanation'])
+        else:
+            st.warning("Detailed analysis not available for this paper.")
+            
+        # Footer
+        st.markdown("---")
+        st.caption("PaperFlux - Powered by Gemini")

pyproject.toml

@@ -10,7 +10,6 @@ license = {text = "MIT"}
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
-    "gradio (>=5.16.0,<6.0.0)",
     "google-generativeai (>=0.8.4,<0.9.0)",
     "redis (>=5.2.1,<6.0.0)",
     "python-dotenv (>=1.0.1,<2.0.0)",
@@ -23,7 +22,9 @@ dependencies = [
     "pymongo (>=4.11.1,<5.0.0)",
     "flask (>=3.1.0,<4.0.0)",
     "tqdm (>=4.67.1,<5.0.0)",
-    "aiohttp (>=3.11.12,<4.0.0)"
+    "aiohttp (>=3.11.12,<4.0.0)",
+    "dotenv (>=0.9.9,<0.10.0)",
+    "streamlit (>=1.42.2,<2.0.0)"
 ]