Phase 3 complete: Backend API for sentence-level analysis

- Update /api/analyze to support sentence-level (use_sentences flag)
- Add /api/update-sentence-category endpoint
- Automatically create training examples from sentence corrections
- Recalculate submission primary category from sentences

app/routes/admin.py

@@ -1,5 +1,5 @@
 from flask import Blueprint, render_template, request, redirect, url_for, session, flash, jsonify, send_file
-from app.models.models import Token, Submission, Settings, TrainingExample, FineTuningRun
+from app.models.models import Token, Submission, Settings, TrainingExample, FineTuningRun, SubmissionSentence
 from app import db
 from app.analyzer import get_analyzer
 from functools import wraps
@@ -277,6 +277,57 @@ def update_category(submission_id):
         print(f"Error updating category: {str(e)}")
         return jsonify({'success': False, 'error': str(e)}), 500
 
+@bp.route('/api/update-sentence-category/<int:sentence_id>', methods=['POST'])
+@admin_required
+def update_sentence_category(sentence_id):
+    """Update category for a specific sentence"""
+    try:
+        sentence = SubmissionSentence.query.get_or_404(sentence_id)
+        data = request.json
+        new_category = data.get('category')
+        
+        # Store original
+        original_category = sentence.category
+        
+        # Validate category
+        if new_category and new_category not in CATEGORIES:
+            return jsonify({'success': False, 'error': f'Invalid category: {new_category}'}), 400
+        
+        # Update sentence
+        sentence.category = new_category
+        
+        # Create/update training example for this sentence
+        if new_category:
+            existing = TrainingExample.query.filter_by(sentence_id=sentence_id).first()
+            
+            if existing:
+                existing.original_category = original_category
+                existing.corrected_category = new_category
+                existing.correction_timestamp = datetime.utcnow()
+            else:
+                training_example = TrainingExample(
+                    sentence_id=sentence_id,
+                    submission_id=sentence.submission_id,
+                    message=sentence.text,  # Just the sentence text
+                    original_category=original_category,
+                    corrected_category=new_category,
+                    contributor_type=sentence.submission.contributor_type
+                )
+                db.session.add(training_example)
+        
+        # Update parent submission's primary category (recalculate from sentences)
+        submission = sentence.submission
+        submission.category = submission.get_primary_category()
+        
+        db.session.commit()
+        
+        return jsonify({'success': True, 'category': new_category})
+    
+    except Exception as e:
+        db.session.rollback()
+        logger.error(f"Error updating sentence category: {str(e)}")
+        return jsonify({'success': False, 'error': str(e)}), 500
+
 @bp.route('/api/toggle-flag/<int:submission_id>', methods=['POST'])
 @admin_required
 def toggle_flag(submission_id):
@@ -298,12 +349,17 @@ def delete_submission(submission_id):
 def analyze_submissions():
     data = request.json
     analyze_all = data.get('analyze_all', False)
+    use_sentences = data.get('use_sentences', True)  # NEW: sentence-level flag (default: True)
 
     # Get submissions to analyze
     if analyze_all:
         to_analyze = Submission.query.all()
     else:
-        to_analyze = Submission.query.filter_by(category=None).all()
+        # For sentence-level, look for submissions without sentence analysis
+        if use_sentences:
+            to_analyze = Submission.query.filter_by(sentence_analysis_done=False).all()
+        else:
+            to_analyze = Submission.query.filter_by(category=None).all()
 
     if not to_analyze:
         return jsonify({'success': False, 'error': 'No submissions to analyze'}), 400
@@ -316,13 +372,38 @@ def analyze_submissions():
 
     for submission in to_analyze:
         try:
-            # Use the free Hugging Face model for classification
-            category = analyzer.analyze(submission.message)
-            submission.category = category
+            if use_sentences:
+                # NEW: Sentence-level analysis
+                sentence_results = analyzer.analyze_with_sentences(submission.message)
+                
+                # Clear old sentences for this submission
+                SubmissionSentence.query.filter_by(submission_id=submission.id).delete()
+                
+                # Create new sentence records
+                for idx, result in enumerate(sentence_results):
+                    sentence = SubmissionSentence(
+                        submission_id=submission.id,
+                        sentence_index=idx,
+                        text=result['text'],
+                        category=result['category'],
+                        confidence=result.get('confidence')
+                    )
+                    db.session.add(sentence)
+                
+                submission.sentence_analysis_done = True
+                # Set primary category for backward compatibility
+                submission.category = submission.get_primary_category()
+                
+                logger.info(f"Analyzed submission {submission.id} into {len(sentence_results)} sentences")
+            else:
+                # OLD: Submission-level analysis (backward compatible)
+                category = analyzer.analyze(submission.message)
+                submission.category = category
+            
             success_count += 1
 
         except Exception as e:
-            print(f"Error analyzing submission {submission.id}: {e}")
+            logger.error(f"Error analyzing submission {submission.id}: {e}")
             error_count += 1
             continue
 
@@ -331,7 +412,8 @@ def analyze_submissions():
     return jsonify({
         'success': True,
         'analyzed': success_count,
-        'errors': error_count
+        'errors': error_count,
+        'sentence_level': use_sentences
     })
 
 @bp.route('/export/json')