Update components/student_flow.py

components/student_flow.py

@@ -4,6 +4,10 @@ from generator import model_manager
 from components.file_processor import get_student_content_input
 from components.export_handler import generate_pdf
 from components.session_manager import update_session_state
+import re
+import logging
+
+logger = logging.getLogger(__name__)
 
 def render_student_flow():
     """Render the student content generation flow"""
@@ -288,86 +292,241 @@ def generate_chunked_content(content_text, student_level, specific_help, filenam
         st.info("🔄 Trying alternative processing method...")
         generate_single_large_content(content_text, student_level, specific_help, filename)
 
-def chunk_content(content, max_chunk_size=8000):
-    """Split content into manageable chunks with robust error handling"""
-    if not content or len(content.strip()) == 0:
-        return []
-    
-    # If content is already smaller than max chunk size, return it as a single chunk
-    if len(content) <= max_chunk_size:
-        return [content.strip()]
-    
-    # Define minimum chunk size (you can adjust this value)
-    min_chunk_size = 500  # Add this line to define the variable
-    
-    # First try: split by paragraphs
-    paragraphs = re.split(r'\n\s*\n', content)
-    paragraphs = [p.strip() for p in paragraphs if p.strip()]
-    
-    chunks = []
-    current_chunk = ""
-    
-    if paragraphs:
-        for paragraph in paragraphs:
-            # If a single paragraph is too large, split it by sentences
-            if len(paragraph) > max_chunk_size:
-                # If we have accumulated content, save it first
-                if current_chunk:
-                    chunks.append(current_chunk.strip())
-                    current_chunk = ""
-                
-                # Split the large paragraph by sentences
-                sentences = re.split(r'[.!?]+', paragraph)
-                sentences = [s.strip() for s in sentences if s.strip()]
+def is_academic_section_start(line):
+    """Detect common research paper section headers with error handling"""
+    try:
+        if not line or not isinstance(line, str):
+            return False
+            
+        academic_headers = [
+            'abstract', 'introduction', 'methods', 'methodology', 'materials and methods',
+            'results', 'findings', 'discussion', 'conclusion', 'conclusions',
+            'references', 'bibliography', 'acknowledgments', 'acknowledgements',
+            'background', 'literature review', 'theoretical framework',
+            'data analysis', 'statistical analysis', 'limitations', 'future work',
+            'implications', 'key findings', 'summary', 'objectives'
+        ]
+        
+        line_lower = line.lower().strip()
+        
+        # Return False for very short lines that are likely not section headers
+        if len(line_lower) < 3 or len(line_lower) > 200:
+            return False
+        
+        # Remove common numbering patterns: "1. Introduction" -> "Introduction"
+        line_clean = re.sub(r'^\d+[\.\)]\s*', '', line_lower)  # "1. Introduction"
+        line_clean = re.sub(r'^[IVX]+[\.\)]\s*', '', line_clean)  # "I. Introduction"
+        line_clean = re.sub(r'^[A-Z]\)\s*', '', line_clean)  # "A) Introduction"
+        
+        # Clean up extra whitespace
+        line_clean = re.sub(r'\s+', ' ', line_clean).strip()
+        
+        # Check if line starts with or contains academic headers
+        for header in academic_headers:
+            try:
+                if (line_clean.startswith(header) or 
+                    f"\n{header}" in f"\n{line_clean}" or
+                    re.match(rf'^{header}[\s:]', line_clean) or
+                    line_clean == header):
+                    logger.debug(f"Detected academic section: '{line_clean}' matches '{header}'")
+                    return True
+            except re.error as e:
+                logger.warning(f"Regex error for header '{header}': {e}")
+                continue
                 
-                for sentence in sentences:
-                    if len(current_chunk) + len(sentence) < max_chunk_size:
-                        current_chunk += sentence + ". "
-                    else:
+        return False
+        
+    except Exception as e:
+        logger.error(f"Error in is_academic_section_start: {e}")
+        return False
+
+def chunk_content(content, max_chunk_size=8000, min_chunk_size=500):
+    """Split content into manageable chunks with robust error handling"""
+    try:
+        # Input validation
+        if not content or not isinstance(content, str):
+            logger.warning("Invalid content provided to chunk_content")
+            return []
+        
+        content = content.strip()
+        if not content:
+            return []
+        
+        # Parameter validation
+        if not isinstance(max_chunk_size, int) or max_chunk_size <= 0:
+            logger.warning(f"Invalid max_chunk_size: {max_chunk_size}, using default 8000")
+            max_chunk_size = 8000
+            
+        if not isinstance(min_chunk_size, int) or min_chunk_size <= 0:
+            logger.warning(f"Invalid min_chunk_size: {min_chunk_size}, using default 500")
+            min_chunk_size = 500
+            
+        if min_chunk_size >= max_chunk_size:
+            logger.warning(f"min_chunk_size ({min_chunk_size}) >= max_chunk_size ({max_chunk_size}), adjusting")
+            min_chunk_size = max(max_chunk_size // 4, 100)
+        
+        # If content is already smaller than max chunk size, return it as a single chunk
+        if len(content) <= max_chunk_size:
+            logger.debug(f"Content fits in single chunk: {len(content)} chars")
+            return [content]
+        
+        logger.info(f"Chunking content: {len(content)} chars, max_chunk_size: {max_chunk_size}")
+        
+        # First try: split by paragraphs
+        paragraphs = []
+        try:
+            paragraphs = re.split(r'\n\s*\n', content)
+            paragraphs = [p.strip() for p in paragraphs if p.strip()]
+        except Exception as e:
+            logger.error(f"Error splitting paragraphs: {e}")
+            # Fallback: split by double newlines
+            paragraphs = [p.strip() for p in content.split('\n\n') if p.strip()]
+        
+        chunks = []
+        current_chunk = ""
+        
+        if paragraphs:
+            logger.debug(f"Processing {len(paragraphs)} paragraphs")
+            
+            for i, paragraph in enumerate(paragraphs):
+                try:
+                    # Skip empty paragraphs
+                    if not paragraph or len(paragraph.strip()) == 0:
+                        continue
+                    
+                    # Check if this paragraph starts a new academic section
+                    is_new_section = False
+                    try:
+                        first_line = paragraph.split('\n')[0] if '\n' in paragraph else paragraph
+                        is_new_section = is_academic_section_start(first_line)
+                    except Exception as e:
+                        logger.warning(f"Error detecting academic section: {e}")
+                    
+                    # If we detect a new academic section and have existing content, consider starting new chunk
+                    if (is_new_section and current_chunk and 
+                        len(current_chunk) > min_chunk_size):
+                        # Only start new chunk if current chunk is substantial
+                        chunks.append(current_chunk.strip())
+                        current_chunk = ""
+                        logger.debug(f"Started new chunk at academic section: {first_line[:50]}...")
+                    
+                    # If a single paragraph is too large, split it by sentences
+                    if len(paragraph) > max_chunk_size:
+                        logger.debug(f"Splitting large paragraph: {len(paragraph)} chars")
+                        
+                        # If we have accumulated content, save it first
                         if current_chunk:
                             chunks.append(current_chunk.strip())
-                        current_chunk = sentence + ". "
-            else:
-                if len(current_chunk) + len(paragraph) < max_chunk_size:
-                    current_chunk += paragraph + "\n\n"
+                            current_chunk = ""
+                        
+                        # Split the large paragraph by sentences
+                        sentences = []
+                        try:
+                            sentences = re.split(r'[.!?]+', paragraph)
+                            sentences = [s.strip() for s in sentences if s.strip()]
+                        except Exception as e:
+                            logger.error(f"Error splitting sentences: {e}")
+                            # Emergency fallback: split by fixed size
+                            sentences = [paragraph[j:j+500] for j in range(0, len(paragraph), 500)]
+                        
+                        for sentence in sentences:
+                            if not sentence:
+                                continue
+                                
+                            if len(current_chunk) + len(sentence) < max_chunk_size:
+                                current_chunk += sentence + ". "
+                            else:
+                                if current_chunk:
+                                    chunks.append(current_chunk.strip())
+                                current_chunk = sentence + ". "
+                    else:
+                        # Normal paragraph processing
+                        if len(current_chunk) + len(paragraph) < max_chunk_size:
+                            current_chunk += paragraph + "\n\n"
+                        else:
+                            if current_chunk:
+                                chunks.append(current_chunk.strip())
+                            current_chunk = paragraph + "\n\n"
+                            
+                except Exception as e:
+                    logger.error(f"Error processing paragraph {i}: {e}")
+                    # Continue with next paragraph instead of failing completely
+                    continue
+                    
+        else:
+            # Fallback: split by sentences if no paragraphs found
+            logger.debug("No paragraphs found, falling back to sentence splitting")
+            sentences = []
+            try:
+                sentences = re.split(r'[.!?]+', content)
+                sentences = [s.strip() for s in sentences if s.strip()]
+            except Exception as e:
+                logger.error(f"Error in sentence splitting fallback: {e}")
+                # Last resort: fixed-size chunks
+                return [content[i:i+max_chunk_size] for i in range(0, len(content), max_chunk_size)]
+            
+            for sentence in sentences:
+                if not sentence:
+                    continue
+                    
+                if len(current_chunk) + len(sentence) < max_chunk_size:
+                    current_chunk += sentence + ". "
                 else:
                     if current_chunk:
                         chunks.append(current_chunk.strip())
-                    current_chunk = paragraph + "\n\n"
-    else:
-        # If no paragraphs found, split by sentences
-        sentences = re.split(r'[.!?]+', content)
-        sentences = [s.strip() for s in sentences if s.strip()]
+                    current_chunk = sentence + ". "
+        
+        # Add the last chunk if it exists
+        if current_chunk.strip():
+            chunks.append(current_chunk.strip())
+        
+        # Final validation and cleanup
+        validated_chunks = []
+        for i, chunk in enumerate(chunks):
+            try:
+                chunk = chunk.strip()
+                if not chunk:
+                    continue
+                    
+                # Merge very small chunks with previous chunk (if exists and not too large)
+                if (len(chunk) < min_chunk_size and validated_chunks and 
+                    len(validated_chunks[-1]) + len(chunk) < max_chunk_size * 1.5):
+                    validated_chunks[-1] = validated_chunks[-1] + "\n\n" + chunk
+                    logger.debug(f"Merged small chunk {i} with previous chunk")
+                
+                # Split chunks that are way too large (emergency fallback)
+                elif len(chunk) > max_chunk_size * 1.5:
+                    logger.warning(f"Chunk {i} too large ({len(chunk)} chars), emergency splitting")
+                    # Split by fixed size as last resort
+                    for j in range(0, len(chunk), max_chunk_size):
+                        sub_chunk = chunk[j:j + max_chunk_size]
+                        if sub_chunk.strip():
+                            validated_chunks.append(sub_chunk.strip())
+                
+                else:
+                    validated_chunks.append(chunk)
+                    
+            except Exception as e:
+                logger.error(f"Error validating chunk {i}: {e}")
+                # Include the chunk anyway to avoid data loss
+                if chunk:
+                    validated_chunks.append(chunk)
+        
+        # Final check - ensure we have chunks
+        if not validated_chunks:
+            logger.warning("No chunks created, returning original content as single chunk")
+            return [content[:max_chunk_size]]
+        
+        logger.info(f"Successfully created {len(validated_chunks)} chunks from {len(content)} chars")
+        return validated_chunks
+        
+    except Exception as e:
+        logger.error(f"Critical error in chunk_content: {e}")
+        # Last resort fallback - return content as single chunk
+        if content and isinstance(content, str):
+            return [content[:max_chunk_size]]
+        return []
         
-        for sentence in sentences:
-            if len(current_chunk) + len(sentence) < max_chunk_size:
-                current_chunk += sentence + ". "
-            else:
-                if current_chunk:
-                    chunks.append(current_chunk.strip())
-                current_chunk = sentence + ". "
-    
-    # Add the last chunk if it exists
-    if current_chunk.strip():
-        chunks.append(current_chunk.strip())
-    
-    # Final validation - ensure no chunk is too large
-    validated_chunks = []
-    for chunk in chunks:
-        if len(chunk) < min_chunk_size and validated_chunks:  # Now min_chunk_size is defined
-            # Merge with previous chunk if too small (unless it's the first chunk)
-            validated_chunks[-1] = validated_chunks[-1] + "\n\n" + chunk
-        elif len(chunk) > max_chunk_size * 1.2:  # Allow 20% overflow
-            # Emergency split by fixed size
-            for i in range(0, len(chunk), max_chunk_size):
-                sub_chunk = chunk[i:i + max_chunk_size]
-                if sub_chunk.strip():
-                    validated_chunks.append(sub_chunk.strip())
-        else:
-            validated_chunks.append(chunk)
-    
-    return validated_chunks
-
 def build_phi3_chunk_prompt(chunk, student_level, specific_help, chunk_num, total_chunks):
     """Build Phi-3 specific prompt for a single chunk - STRICTER VERSION"""
     prompt = f"""TASK: Write the actual simplified explanation for this content section.