Add research attention analysis endpoint with real CodeGen tokenization

- Implement /analyze/research/attention endpoint
- Extract real token IDs from CodeGen tokenizer
- Track attention weights across all 20 layers per generation step
- Return top-k token alternatives with probabilities
- Store per-step attention data for token-by-token analysis

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

backend/model_service.py

@@ -1526,10 +1526,10 @@ async def analyze_research_attention(request: Dict[str, Any], authenticated: boo
         # Build response
         response = {
             "prompt": prompt,
-            "promptTokens": [{"text": t, "idx": i, "bytes": len(t.encode('utf-8')), "type": "prompt"}
-                           for i, t in enumerate(prompt_tokens)],
-            "generatedTokens": [{"text": t, "idx": i, "bytes": len(t.encode('utf-8')), "type": "generated"}
-                              for i, t in enumerate(generated_tokens)],
+            "promptTokens": [{"text": t, "idx": tid, "bytes": len(t.encode('utf-8')), "type": "prompt"}
+                           for tid, t in zip(prompt_token_ids, prompt_tokens)],
+            "generatedTokens": [{"text": t, "idx": tid, "bytes": len(t.encode('utf-8')), "type": "generated"}
+                              for tid, t in zip(generated_token_ids, generated_tokens)],
             "tokenAlternatives": token_alternatives_by_step,  # Top-k alternatives for each token
             "layersDataByStep": layer_data_by_token,  # Layer data for ALL generation steps
             "layersData": layer_data_by_token[-1] if layer_data_by_token else [],  # Keep for backward compatibility
@@ -2189,6 +2189,234 @@ async def get_swe_bench_comparison(
 
     return comparison
 
+# ==============================================================================
+# VOCABULARY & TOKENIZATION ENDPOINTS
+# ==============================================================================
+
+@app.post("/vocabulary/search")
+async def search_vocabulary(
+    request: Dict[str, Any],
+    authenticated: bool = Depends(verify_api_key)
+):
+    """Search vocabulary by query string"""
+    query = request.get("query", "").lower()
+    limit = request.get("limit", 50)
+
+    if not query:
+        return {"results": [], "total": 0}
+
+    vocab = manager.tokenizer.get_vocab()
+
+    # Search for tokens containing the query
+    results = []
+    for token, token_id in vocab.items():
+        if query in token.lower():
+            results.append({
+                "token": token,
+                "token_id": token_id,
+                "byte_length": len(token.encode('utf-8'))
+            })
+            if len(results) >= limit:
+                break
+
+    return {
+        "results": results,
+        "total": len(results),
+        "vocabulary_size": len(vocab)
+    }
+
+@app.get("/vocabulary/browse")
+async def browse_vocabulary(
+    page: int = 0,
+    page_size: int = 100,
+    filter_type: str = "all",  # all, programming, common, functions
+    authenticated: bool = Depends(verify_api_key)
+):
+    """Browse vocabulary with pagination and smart filtering"""
+    vocab = manager.tokenizer.get_vocab()
+
+    # Smart filtering for programming tokens
+    if filter_type == "programming":
+        # Python keywords and common programming terms
+        programming_keywords = {
+            "def", "class", "return", "import", "from", "if", "else", "elif",
+            "for", "while", "break", "continue", "pass", "try", "except",
+            "finally", "with", "as", "lambda", "yield", "async", "await",
+            "None", "True", "False", "and", "or", "not", "in", "is"
+        }
+        filtered_vocab = {k: v for k, v in vocab.items() if k in programming_keywords}
+    elif filter_type == "functions":
+        # Common function/method names
+        filtered_vocab = {k: v for k, v in vocab.items()
+                         if any(term in k.lower() for term in ["length", "size", "count", "append", "insert", "remove", "delete", "get", "set", "print", "open", "close", "read", "write"])}
+    elif filter_type == "common":
+        # Most common English words (simple heuristic: short tokens)
+        filtered_vocab = {k: v for k, v in vocab.items() if len(k) <= 4 and k.isalpha()}
+    else:
+        filtered_vocab = vocab
+
+    # Sort by token ID
+    sorted_items = sorted(filtered_vocab.items(), key=lambda x: x[1])
+
+    # Paginate
+    start = page * page_size
+    end = start + page_size
+    page_items = sorted_items[start:end]
+
+    results = []
+    for token, token_id in page_items:
+        results.append({
+            "token": token,
+            "token_id": token_id,
+            "byte_length": len(token.encode('utf-8'))
+        })
+
+    return {
+        "items": results,
+        "total": len(filtered_vocab),
+        "page": page,
+        "page_size": page_size,
+        "total_pages": (len(filtered_vocab) + page_size - 1) // page_size
+    }
+
+@app.post("/tokenize/preview")
+async def tokenize_preview(
+    request: Dict[str, Any],
+    authenticated: bool = Depends(verify_api_key)
+):
+    """Live tokenization preview for arbitrary text"""
+    from .tokenizer_utils import TokenizerMetadata, get_tokenizer_stats
+
+    text = request.get("text", "")
+
+    if not text:
+        return {"tokens": [], "stats": {}}
+
+    # Tokenize
+    token_ids = manager.tokenizer.encode(text, add_special_tokens=False)
+
+    # Get metadata
+    metadata = TokenizerMetadata(manager.tokenizer)
+    token_analysis = metadata.analyze_tokens(token_ids)
+    stats = get_tokenizer_stats(manager.tokenizer, text)
+
+    return {
+        "text": text,
+        "tokens": token_analysis,
+        "stats": stats,
+        "token_count": len(token_ids)
+    }
+
+@app.post("/tokenize/compare")
+async def compare_tokenizers(
+    request: Dict[str, Any],
+    authenticated: bool = Depends(verify_api_key)
+):
+    """Compare tokenization across different models"""
+    from transformers import AutoTokenizer
+    from .tokenizer_utils import get_tokenizer_stats
+
+    text = request.get("text", "")
+    models = request.get("models", ["Salesforce/codegen-350M-mono"])
+
+    if not text:
+        return {"results": {}}
+
+    results = {}
+
+    for model_name in models:
+        try:
+            # Load tokenizer (will be cached by transformers)
+            if model_name == "Salesforce/codegen-350M-mono":
+                tokenizer = manager.tokenizer
+            else:
+                tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+            # Tokenize
+            tokens = tokenizer.tokenize(text)
+            token_ids = tokenizer.encode(text, add_special_tokens=False)
+            token_texts = [tokenizer.decode([tid]) for tid in token_ids]
+            stats = get_tokenizer_stats(tokenizer, text)
+
+            results[model_name] = {
+                "tokens": tokens,
+                "token_ids": token_ids,
+                "token_texts": token_texts,
+                "token_count": len(token_ids),
+                "stats": stats
+            }
+        except Exception as e:
+            logger.error(f"Error loading tokenizer {model_name}: {e}")
+            results[model_name] = {"error": str(e)}
+
+    return {"text": text, "results": results}
+
+@app.post("/token/metadata")
+async def get_token_metadata(
+    request: Dict[str, Any],
+    authenticated: bool = Depends(verify_api_key)
+):
+    """Get comprehensive metadata for a specific token"""
+    from .tokenizer_utils import TokenizerMetadata
+
+    token_id = request.get("token_id")
+
+    if token_id is None:
+        raise HTTPException(status_code=400, detail="token_id is required")
+
+    metadata = TokenizerMetadata(manager.tokenizer)
+
+    # Get token text
+    token_text = manager.tokenizer.decode([token_id])
+
+    # Get BPE pieces
+    bpe_pieces = metadata.get_subword_pieces(token_id)
+
+    # Get byte length
+    byte_length = metadata.get_byte_length(token_id)
+
+    # Check if special token
+    special_tokens = {
+        "eos": manager.tokenizer.eos_token_id,
+        "bos": manager.tokenizer.bos_token_id,
+        "pad": manager.tokenizer.pad_token_id,
+        "unk": manager.tokenizer.unk_token_id
+    }
+    is_special = token_id in special_tokens.values()
+
+    # Check if multi-split (returns array, extract first element)
+    is_multi_split_array = metadata.is_multi_split_identifier([token_id])
+    is_multi_split = is_multi_split_array[0] if is_multi_split_array else False
+
+    # DEBUG LOGGING
+    print(f"\n{'='*60}")
+    print(f"TOKEN METADATA DEBUG - Token ID: {token_id}")
+    print(f"{'='*60}")
+    print(f"Token Text: {repr(token_text)}")
+    print(f"BPE Pieces: {bpe_pieces}")
+    print(f"Num Pieces: {len(bpe_pieces)}")
+    print(f"Byte Length: {byte_length}")
+    print(f"Is Special: {is_special}")
+    print(f"Multi-split Array: {is_multi_split_array}")
+    print(f"Multi-split Boolean: {is_multi_split} (type: {type(is_multi_split).__name__})")
+    print(f"Tokenizer Type: {metadata.tokenizer_type}")
+    print(f"{'='*60}\n")
+
+    result = {
+        "token_id": token_id,
+        "text": token_text,
+        "bpe_pieces": bpe_pieces,
+        "byte_length": byte_length,
+        "is_special": is_special,
+        "is_multi_split": is_multi_split,
+        "num_pieces": len(bpe_pieces),
+        "tokenizer_type": metadata.tokenizer_type
+    }
+
+    print(f"RESPONSE: {result}\n")
+
+    return result
+
 if __name__ == "__main__":
     import uvicorn
     uvicorn.run(app, host="0.0.0.0", port=8000)