nodes/node4.py

import json
import re


from core.state import AgenticState
from loguru import logger


@logger.catch
async def node_4_intelligent_structuring_api(state: AgenticState) -> AgenticState:

    logger.info("🚀 Node 4: Intelligent Structuring started")

    transcript = state.cleaned_transcript or ""
    

    
    # Debug: Check if state has other expected fields
    logger.info(f"   - State summary: url={state.youtube_url}, transcript_len={len(state.cleaned_transcript)}")

    # Or convert to dict with safe values
    safe_state = {
        "youtube_url": state.youtube_url,
        "video_id": state.video_id,
        "transcript_len": len(state.cleaned_transcript),
        "use_api": state.use_api_for_structuring
    }
    logger.info(f"   - State: {safe_state}")
    
    # Debug: Check for errors in state
    if isinstance(state, dict):
        errors = state.get('errors', [])

    else:
        errors = getattr(state, 'errors', [])
    
    if errors:
        logger.error("⚠️ Existing errors in state")
        for i, error in enumerate(errors):
            logger.error(
                "Node 4: Existing error #{i} → {msg}",
                i=i,
                msg=error.get("message", str(error)) if isinstance(error, dict) else str(error)
            )


    


    if not transcript:
        state.errors.append({"type": "no_transcript"})
        logger.error("Node 4: ⚠️ No transcript")
        return state


    llm = state.llm 
    if not llm:
        state.errors.append("LLM not available in state")
        logger.error("Node 4: LLM not available in state")
        return state
    
    # Config
    

    MODEL_LIMIT = 7500 # Save zone
    PROMPT_TOKENS = 1800
    CHARS_PER_TOKEN = 3.5

    MAX_CHARS = int((MODEL_LIMIT - PROMPT_TOKENS) * CHARS_PER_TOKEN)

    OVERLAP = 800
    MAX_CHUNKS = 40


    
    # Save json parse
    

    def safe_json(text):

        if "```" in text:
            parts = text.split("```")
            text = parts[1]
            if text.startswith("json"):
                text = text[4:]

        text = text.strip()

        try:
            return json.loads(text)
        except:
            # fallback extraction
            match = re.search(r"\{.*\}", text, re.DOTALL)
            if match:
                return json.loads(match.group())
            raise


    
    # Save chunking
    

    logger.info("Creating chunks")

    chunks = []

    pos = 0
    length = len(transcript)

    while pos < length and len(chunks) < MAX_CHUNKS:

        end = min(pos + MAX_CHARS, length)

        if end < length:

            boundary = transcript.rfind(". ", pos, end)

            if boundary != -1 and boundary > pos:
                end = boundary + 2

        chunk = transcript[pos:end]

        chunks.append(chunk)

        new_pos = end - OVERLAP

        if new_pos <= pos:
            new_pos = end

        pos = new_pos

    logger.info("   chunks created: {chunk_count}", chunk_count=len(chunks))


    
    # Map step
    

    chunk_summaries = []
    sections = []
    quotes = []
    entities = []
    topics = []

    for i, chunk in enumerate(chunks):

        logger.info(
            "   analyzing chunk {current}/{total}",
            current=i + 1,
            total=len(chunks)
        )

        prompt = f"""
You are analyzing a segment of a long podcast transcript.

Extract meaningful structure and ideas.

Return JSON only.

{{
 "chunk_summary":"4-5 sentence explanation of the main ideas",
 "sections":[
  {{
   "title":"descriptive section title",
   "summary":"3 sentence explanation",
   "key_points":[
    "important insight",
    "important insight",
    "important insight",
    "important insight"
   ]
  }}
 ],
 "quotes":["memorable quote from speaker"],
 "entities":["people companies technologies books"],
 "topics":["specific conceptual topics discussed"]
}}

Rules:
- focus on meaningful ideas
- avoid generic phrases
- insights must be specific
- section titles must describe the topic

TRANSCRIPT:
{chunk}
"""

        try:

            response = llm.invoke(prompt)

            text = response.content if hasattr(response,"content") else str(response)

            data = safe_json(text)

        except Exception as e:

            logger.opt(exception=e, diagnose=False).error("Node 4: ⚠️ chunk failed")
            continue


        chunk_summaries.append(data.get("chunk_summary",""))

        sections.extend(data.get("sections",[]))
        quotes.extend(data.get("quotes",[]))
        entities.extend(data.get("entities",[]))
        topics.extend(data.get("topics",[]))


    
    # Global reduce
    

    logger.info("Building global structure")

    summary_text = "\n".join(chunk_summaries[:30])

    reduce_prompt = f"""
These are summaries of segments from a long podcast.

{summary_text}

Your task:

Create the GLOBAL structure of the full conversation.

Return JSON:

{{
 "executive_summary":"8 sentence explanation of the entire episode",

 "sections":[
  {{
   "title":"section title",
   "summary":"4 sentence summary",
   "key_points":[
    "insight",
    "insight",
    "insight",
    "insight"
   ]
  }}
 ]
}}

Rules:
- produce 8 to 12 sections
- titles must reflect the real discussion topics
- insights must be concrete and specific
"""

    try:

        response = llm.invoke(reduce_prompt)

        text = response.content if hasattr(response,"content") else str(response)

        data = safe_json(text)

        final_sections = data.get("sections",[])[:14]
        executive_summary = data.get("executive_summary","")

    except Exception as e:


        logger.opt(exception=e, diagnose=False).error("Node 4: ⚠️ reduce step failed")

        final_sections = sections[:14]
        executive_summary = ""


    
    # Deput helper
    

    def dedup(lst, limit):

        out = []

        for x in lst:

            x = str(x).strip()

            if not x:
                continue

            if x not in out:
                out.append(x)

            if len(out) >= limit:
                break

        return out


    def dedup_quotes(qs):

        out = []

        for q in qs:

            q = str(q).strip()

            if len(q) < 20:
                continue

            duplicate = False

            for e in out:

                w1 = set(q.lower().split())
                w2 = set(e.lower().split())

                if w1 and w2:

                    overlap = len(w1 & w2) / max(len(w1), len(w2))

                    if overlap > 0.75:
                        duplicate = True
                        break

            if not duplicate:
                out.append(q)

            if len(out) >= 10:
                break

        return out


    
    # Topic consolidation
    

    topics = dedup(topics, 30)

    topic_prompt = f"""
These are topics extracted from a podcast.

{topics}

Group and consolidate them into the 12 most important conceptual topics.

Return JSON:

{{
 "topics":["topic","topic","topic"]
}}
"""

    try:

        response = llm.invoke(topic_prompt)

        text = response.content if hasattr(response,"content") else str(response)

        data = safe_json(text)

        topics = data.get("topics", topics)

    except:

        topics = topics


    
    # Final structure
    

    structured = {

        "executive_summary": executive_summary,

        "sections": final_sections,

        "chapter_list":[
            {"title": s["title"], "start_time": None}
            for s in final_sections
        ],

        "key_quotes": dedup_quotes(quotes),

        "mentioned_entities": dedup(entities, 30),

        "main_topics": topics
    }


    
    # Write state
    

    state.structured_script = structured
    state.chapter_list = structured["chapter_list"]
    state.key_quotes = structured["key_quotes"]
    state.mentioned_entities = structured["mentioned_entities"]
    state.main_topics = structured["main_topics"]


    logger.info("\n✅ Node 4 finished")

    logger.info("   Sections: {count}", count=len(structured["sections"]))
    logger.info("   Quotes: {count}", count=len(structured["key_quotes"]))
    logger.info("   Topics: {count}", count=len(structured["main_topics"]))
    logger.info("   Entities: {count}", count=len(structured["mentioned_entities"]))

    return state