Tests are not working.

CODE (I've obfuscated keys).
import asyncio
import json
import os
from typing import List, Dict, Any, Tuple, Optional
from datetime import datetime, timezone, timedelta
from twscrape import API, gather
from http.cookiejar import MozillaCookieJar
import anthropic
import html
import logging
from openai import OpenAI
import yt_dlp
import time
import sys
import argparse
import shutil
import re
import warnings
from tenacity import retry, wait_exponential, stop_after_attempt

# Set up logging to both console and file
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler("tweetreader.log"),
        logging.StreamHandler(sys.stdout)
    ]
)
logger = logging.getLogger(__name__)

logger.info("Script started")

COOKIE_FILE = 'cookies.txt'
CACHE_FILE = 'tweet_cache.json'
YOUTUBE_CACHE_DIR = 'youtube_cache'
CLAUDE_API_KEY = key1 
OpenAI_API_KEY = key2 

client_claude = anthropic.Client(api_key=CLAUDE_API_KEY)
OpenAI.api_key = OpenAI_API_KEY

client = OpenAI(api_key=OpenAI_API_KEY)

logger.info("API clients initialized")

MODEL_PROVIDER_MAPPING: Dict[str, str] = {
    "gpt-4o-mini": "openai",
    "gpt-4": "openai",
    "gpt-3.5-turbo": "openai",
    "claude-3-opus-20240229": "claude",
    "claude-3-sonnet-20240229": "claude",
    "claude-3-haiku-20240307": "claude",
    "claude-2.1": "claude"
}

FRIENDLY_NAME_MAPPING: Dict[str, str] = {
    "gpt": "gpt-4o-mini",
    "claude": "claude-3-sonnet-20240229"
}

def get_provider_from_model(model: str) -> Optional[str]:
    return MODEL_PROVIDER_MAPPING.get(model)

def resolve_model_friendly_name(model: Optional[str]) -> Optional[str]:
    if model in FRIENDLY_NAME_MAPPING:
        return FRIENDLY_NAME_MAPPING[model]
    return model

def validate_model_provider(model: Optional[str], provider: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
    model = resolve_model_friendly_name(model)
    if model and not provider:
        provider = get_provider_from_model(model)
    elif provider and not model:
        model = FRIENDLY_NAME_MAPPING.get(provider.lower(), None)
    elif model and provider:
        if get_provider_from_model(model) != provider:
            raise ValueError(f"Mismatch between specified model '{model}' and provider '{provider}'")
    # Add validation to ensure provider is not None
    if model and not provider:
        raise ValueError(f"Provider could not be determined for model '{model}'. Please check the model name.")
    return model, provider

@retry(wait=wait_exponential(multiplier=1, min=4, max=10), stop=stop_after_attempt(5))
def call_anthropic_api(prompt, model, provider):
    client = anthropic.Client(api_key=CLAUDE_API_KEY)
    response = client.completion(
        model=model,
        max_tokens=2000,
        prompt=prompt
    )
    return response['completion']

def get_youtube_cache_key(video_id: str, model: str) -> str:
    return f"{video_id}_{model}_subtitles.json"

def load_youtube_cache(cache_dir: str, video_id: str, model: str) -> Optional[Dict[str, Any]]:
    cache_file = os.path.join(cache_dir, get_youtube_cache_key(video_id, model))
    if os.path.exists(cache_file):
        try:
            with open(cache_file, 'r') as f:
                return json.load(f)
        except json.JSONDecodeError:
            return None
    return None

def save_youtube_cache(cache_dir: str, video_id: str, model: str, data: Dict[str, Any]) -> None:
    cache_file = os.path.join(cache_dir, get_youtube_cache_key(video_id, model))
    with open(cache_file, 'w') as f:
        json.dump(data, f, indent=2)

def purge_cache(purge_option: Optional[str]) -> None:
    if not purge_option:
        return

    if purge_option in ['all', 'tweets']:
        logger.info("Purging tweet analysis cache")
        if os.path.exists(CACHE_FILE):
            try:
                with open(CACHE_FILE, 'r') as f:
                    cache = json.load(f)
                cache['openai_analyses'] = {}
                cache['claude_analyses'] = {}
                with open(CACHE_FILE, 'w') as f:
                    json.dump(cache, f, indent=2)
            except (IOError, json.JSONDecodeError) as e:
                logger.error(f"Error purging tweet cache: {str(e)}")

    if purge_option in ['all', 'yt-all', 'yt-summary']:
        logger.info("Purging YouTube summary cache")
    if os.path.exists(YOUTUBE_CACHE_DIR):
        for filename in os.listdir(YOUTUBE_CACHE_DIR):
            if filename.endswith('_summary.json'):
                file_path = os.path.join(YOUTUBE_CACHE_DIR, filename)
                try:
                    os.unlink(file_path)
                except Exception as e:
                    logger.error(f"Failed to delete {file_path}. Reason: {str(e)}")
                    
    if purge_option in ['all', 'yt-all']:
        logger.info("Purging YouTube subtitle cache")
        if os.path.exists(YOUTUBE_CACHE_DIR):
            for filename in os.listdir(YOUTUBE_CACHE_DIR):
                if not filename.endswith('_summary.json'):
                    file_path = os.path.join(YOUTUBE_CACHE_DIR, filename)
                    try:
                        if os.path.isfile(file_path) or os.path.islink(file_path):
                            os.unlink(file_path)
                        elif os.path.isdir(file_path):
                            shutil.rmtree(file_path)
                    except Exception as e:
                        logger.error(f"Failed to delete {file_path}. Reason: {str(e)}")
                        
def parse_netscape_cookies(file_path):
    cookie_jar = MozillaCookieJar(file_path)
    cookie_jar.load(ignore_discard=True, ignore_expires=True)
    cookies = {}
    for cookie in cookie_jar:
        cookies[cookie.name] = cookie.value
    return json.dumps(cookies)

def load_cache():
    if os.path.exists(CACHE_FILE):
        with open(CACHE_FILE, 'r') as f:
            return json.load(f)
    return {}

def save_cache(cache):
    with open(CACHE_FILE, 'w') as f:
        json.dump(cache, f, indent=2)

def format_output(tweet, analysis, toggle_order=False):
    tweet_content = f"""
Account: {tweet['username']}
Date: {tweet['created_at']}
View Tweet
{tweet['text']}

Likes: {tweet['likes']} | Retweets: {tweet['retweets']} | Replies: {tweet['replies']}
"""

    analysis_content = f"""
Analysis:
Openai Analysis:
Trading Position: {analysis.get('trading_position', 'Not applicable')}  
Instruments: {analysis.get('instruments', 'Not applicable')}  
Levels: {analysis.get('levels', 'Not applicable')}
"""

    if toggle_order:
        return f"{analysis_content}\n{tweet_content}"
    else:
        return f"{tweet_content}\n{analysis_content}"    
def format_tweet(tweet_data):
    created_at = tweet_data.get('date', '')
    if isinstance(created_at, datetime):
        created_at = created_at.isoformat()
    elif not isinstance(created_at, str):
        created_at = str(created_at)

    media = tweet_data.get('media', [])
    images = []
    if isinstance(media, list):
        for entry in media:
            url = entry.get('url') or entry.get('thumbnailUrl')
            if url:
                images.append(url)
                logger.info(f"Added image URL: {url}")
            else:
                logger.warning(f"No URL found for media entry: {entry}")

    logger.info(f"Extracted {len(images)} image URLs for tweet {tweet_data.get('id', 0)}")

    return {
        "id": tweet_data.get('id', 0),
        "created_at": created_at,
        "text": tweet_data.get('rawContent', ''),
        "likes": tweet_data.get('likeCount', 0),
        "retweets": tweet_data.get('retweetCount', 0),
        "replies": tweet_data.get('replyCount', 0),
        "images": images,
        "pinned": tweet_data.get('pinned', False)
    }
        
async def fetch_and_cache_tweets(api, username, limit=15):
    logger.info(f"Fetching tweets for {username} with limit {limit}")
    cache = load_cache()
    user_cache = cache.get(username, {'last_fetched': None, 'tweets': {}, 'claude_analyses': {}})

    current_time = datetime.now(timezone.utc)

    last_fetched = datetime(1970, 1, 1, tzinfo=timezone.utc)
    if user_cache['last_fetched'] is not None:
        last_fetched = datetime.fromisoformat(user_cache['last_fetched']).replace(tzinfo=timezone.utc)

    cache_is_fresh = current_time - last_fetched < timedelta(hours=1)

    if cache_is_fresh:
        logger.info(f"Using cached tweets for {username}")
        all_tweets = list(user_cache['tweets'].values())
    else:
        logger.info(f"Fetching new tweets for {username}")
        try:
            user = await api.user_by_login(username)
            if user is None:
                logger.error(f"User {username} not found")
                return [], user_cache['claude_analyses']

            new_tweets = []
            async for tweet in api.user_tweets(user.id, limit=limit):
                tweet_data = format_tweet(tweet.__dict__)
                if str(tweet_data['id']) not in user_cache['tweets']:
                    user_cache['tweets'][str(tweet_data['id'])] = tweet_data
                    new_tweets.append(tweet_data)

                if len(new_tweets) >= limit:
                    break

            if new_tweets:
                logger.info(f"Fetched {len(new_tweets)} new tweets for {username}")
                user_cache['last_fetched'] = current_time.isoformat()
                cache[username] = user_cache
                save_cache(cache)
            else:
                logger.info(f"No new tweets for {username}")

            all_tweets = list(user_cache['tweets'].values())
        except Exception as e:
            logger.exception(f"Error fetching tweets for {username}: {e}")
            all_tweets = list(user_cache['tweets'].values())

    all_tweets.sort(key=lambda x: datetime.fromisoformat(x['created_at']).replace(tzinfo=timezone.utc), reverse=True)

    return all_tweets[:limit], user_cache.get('claude_analyses', {})
def fetch_channel_id(channel_url):
    logger.info(f"Fetching channel ID for URL: {channel_url}")
    try:
        ydl_opts = {
            'quiet': True,
            'no_warnings': True,
            'playlist_items': '0',
            'extract_flat': True,
        }
        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
            info = ydl.extract_info(channel_url, download=False)
            if info and 'channel_id' in info:
                channel_id = info['channel_id']
                logger.info(f"Successfully fetched channel ID: {channel_id}")
                return channel_id
            else:
                logger.error(f"Failed to fetch channel ID for URL: {channel_url}")
                return None
    except Exception as e:
        logger.exception(f"Error fetching channel ID for URL {channel_url}: {str(e)}")
        return None

import re
import logging

def convert_vtt_to_text(vtt_file_path):
    import re
    try:
        with open(vtt_file_path, 'r', encoding='utf-8') as f:
            lines = f.readlines()

        # Remove WEBVTT header if present
        lines = [line.strip() for line in lines if line.strip() != 'WEBVTT']

        subtitle_lines = []
        for line in lines:
            # Skip empty lines
            if not line:
                continue
            # Skip index numbers
            if re.match(r'^\d+$', line):
                continue
            # Skip timestamps
            if re.match(r'\d{2}:\d{2}:\d{2}.\d{3}\s*-->\s*\d{2}:\d{2}:\d{2}.\d{3}', line):
                continue
            # Anything else is a subtitle line
            subtitle_lines.append(line)

        subtitles_text = '\n'.join(subtitle_lines)
        return subtitles_text.strip()
    except FileNotFoundError:
        logger.error(f"VTT file not found: {vtt_file_path}")
        return ""
def fetch_youtube_subtitles(url, num_videos=1, cache_dir=YOUTUBE_CACHE_DIR, model=None):
    logger.info(f"Fetching YouTube subtitles for URL: {url}")

    ydl_opts = {
        'writesubtitles': True,
        'writeautomaticsub': True,
        'subtitleslangs': ['en'],
        'skip_download': True,
        'outtmpl': os.path.join(cache_dir, '%(id)s.en.vtt'),
        'playlistend': num_videos
    }

    subtitles_list = []
    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
        info = ydl.extract_info(url, download=False)
        entries = info['entries'] if 'entries' in info else [info]
        for entry in entries[:num_videos]:
            video_id = entry['id']
            
            # Check cache for subtitles
            cached_subtitles = load_youtube_cache(cache_dir, video_id, model)
            if cached_subtitles:
                logger.info(f"Using cached subtitles for video ID: {video_id}")
                subtitles_list.append({
                    'video_id': video_id,
                    'title': entry['title'],
                    'subtitles': cached_subtitles['subtitles'],
                    'url': entry['url']
                })
                continue

            subtitle_file = os.path.join(cache_dir, f"{video_id}.en.vtt")
            if os.path.exists(subtitle_file):
                subtitles_text = convert_vtt_to_text(subtitle_file)
                subtitles_list.append({
                    'video_id': video_id,
                    'title': entry['title'],
                    'subtitles': subtitles_text,
                    'url': entry['url']
                })
                
                # Cache the subtitles
                save_youtube_cache(cache_dir, video_id, model, {'subtitles': subtitles_text})
            else:
                logger.warning(f"Subtitle file not found for video ID: {video_id}")
                subtitles_list.append({
                    'video_id': video_id,
                    'title': entry['title'],
                    'subtitles': "",
                    'url': entry['url']
                })

    return subtitles_list

def summarize_content(
    content: str,
    model: Optional[str],
    provider: Optional[str],
    chunk_model: Optional[str],
    chunk_provider: Optional[str],
    max_chunk_size: int = 124000
) -> Dict[str, Any]:
    if not content:
        raise ValueError("Content cannot be empty")
    
    model, provider = validate_model_provider(model, provider)
    chunk_model, chunk_provider = validate_model_provider(chunk_model, chunk_provider)
    
    if not chunk_model:
        chunk_model, chunk_provider = model, provider
    
    if len(content) <= max_chunk_size:
        return summarize_single_chunk(content, model, provider)
    else:
        chunks = [content[i:i + max_chunk_size] for i in range(0, len(content), max_chunk_size)]
        chunk_summaries = [summarize_single_chunk(chunk, chunk_model, chunk_provider) for chunk in chunks]
        return combine_summaries(chunk_summaries, model, provider)

def summarize_single_chunk(chunk: str, model: str, provider: str) -> Dict[str, Any]:
    if not chunk:
        raise ValueError("Chunk cannot be empty")
    
    # Step 1: Generate a detailed summary
    initial_prompt = f"""
    Analyze the following YouTube video subtitles and provide a detailed summary:

    Subtitles:
    {chunk}

    Please provide:
    1. A comprehensive overview of the video content
    2. Key points discussed, including specific strategies, predictions, or analyses mentioned
    3. Notable quotes with timestamps (if available)
    4. Overall sentiment and confidence level of the speaker(s)
    5. Description of any charts, graphs, or visual aids mentioned
    6. A list of cryptocurrencies discussed, their associated price levels, predictions, or analyses

    Format your response as a JSON object.
    """
    
    initial_summary = _get_model_response(initial_prompt, model, provider)
    
    # Step 2: Generate more targeted analysis based on the initial summary
    targeted_prompt = f"""
    Based on the following summary of a cryptocurrency-related video, provide a more in-depth analysis:

    {json.dumps(initial_summary, indent=2)}

    Please provide:
    1. A critical evaluation of the strategies or predictions mentioned
    2. Comparison with current market trends or historical data
    3. Potential risks or limitations of the discussed strategies
    4. Alternative viewpoints or strategies not mentioned in the video
    5. Technical analysis of any specific trading setups or patterns described

    Format your response as a JSON object.
    """
    
    targeted_analysis = _get_model_response(targeted_prompt, model, provider)
    
    # Combine the results
    combined_result = {
        "initial_summary": initial_summary,
        "targeted_analysis": targeted_analysis
    }
    
    return combined_result

def _get_model_response(prompt: str, model: str, provider: str) -> Dict[str, Any]:
    try:
        if provider == 'openai':
            client = OpenAI(api_key=OpenAI_API_KEY)
            response = client.chat.completions.create(
                model=model,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.7,
                max_tokens=2000,
                top_p=1,
                frequency_penalty=0,
                presence_penalty=0
            )
            content = response.choices[0].message.content
        elif provider == 'claude':
            client = anthropic.Client(api_key=CLAUDE_API_KEY)
            response = client.messages.create(
                model=model,
                max_tokens=2000,
                messages=[{"role": "user", "content": prompt}]
            )
            content = response.content[0].text
        else:
            raise ValueError(f"Unsupported provider: {provider}")
        
        # Try to parse the content as JSON
        try:
            return json.loads(content)
        except json.JSONDecodeError:
            # If parsing fails, try to extract JSON from the response
            json_match = re.search(r'\{.*\}', content, re.DOTALL)
            if json_match:
                return json.loads(json_match.group())
            else:
                logging.error(f"Could not extract valid JSON from the response: {content}")
                return {
                    "error": "Failed to parse JSON response",
                    "raw_content": content
                }
    except Exception as e:
        logging.error(f"Error in _get_model_response: {str(e)}")
        return {
            "error": f"An error occurred while getting model response: {str(e)}",
            "raw_content": ""
        }
def combine_summaries(summaries: List[Dict[str, Any]], model: str, provider: str) -> Dict[str, Any]:
    if not summaries:
        raise ValueError("Summaries list cannot be empty")
    
    combined_content = "\n\n".join(json.dumps(summary) for summary in summaries)
    prompt = f"""
    Combine the following summaries into a single coherent summary:

    {combined_content}

    Please provide the following:
    1. A high-level overview of the video content (2-3 sentences)
    2. A deeper dive into the main points discussed (5-7 sentences)
    3. A list of cryptocurrencies mentioned, their associated price levels or predictions, and timestamps (if available)

    Format your response as follows:
    {{
        "high_level_overview": "...",
        "deep_dive": "...",
        "coins_and_levels": [
            ["Bitcoin", "$30,000", "https://youtu.be/VIDEO_ID?t=120"],
            ["Ethereum", "$2,000", "https://youtu.be/VIDEO_ID?t=180"],
            ...
        ]
    }}
    """
    
    return summarize_single_chunk(prompt, model, provider)


from typing import List, Dict, Any, Optional

def analyze_youtube_subtitles(
    subtitles: List[Dict[str, Any]],
    model: Optional[str],
    provider: Optional[str],
    chunk_model: Optional[str],
    chunk_provider: Optional[str],
    cache_dir: str = YOUTUBE_CACHE_DIR
) -> List[Dict[str, Any]]:
    if not subtitles:
        logger.warning("No subtitles provided for analysis")
        return []
    
    summaries = []
    for subtitle in subtitles:
        if not isinstance(subtitle, dict) or 'url' not in subtitle or 'subtitles' not in subtitle:
            logger.warning(f"Invalid subtitle format: {subtitle}")
            continue
        
        video_id = subtitle['url'].split('v=')[-1]
        cache_key = f"{video_id}_{model}_{provider}_{chunk_model}_{chunk_provider}"
        cache_file = os.path.join(cache_dir, f"{cache_key}_summary.json")
        
        try:
            if os.path.exists(cache_file):
                with open(cache_file, 'r') as f:
                    summary = json.load(f)
                logger.info(f"Using cached analysis for video ID: {video_id}")
            else:
                summary = summarize_content(
                    subtitle['subtitles'],
                    model, provider,
                    chunk_model, chunk_provider
                )
                
                # Cache the summary
                with open(cache_file, 'w') as f:
                    json.dump(summary, f, indent=4)
            
            # Add video metadata to the summary
            summary['video_id'] = video_id
            summary['video_url'] = subtitle['url']
            summary['video_title'] = subtitle.get('title', 'Unknown Title')
            
            summaries.append(summary)
        except Exception as e:
            logger.exception(f"Error analyzing subtitles for video ID {video_id}: {str(e)}")
            summaries.append({
                'video_id': video_id,
                'video_url': subtitle['url'],
                'video_title': subtitle.get('title', 'Unknown Title'),
                'error': str(e)
            })
    
    return summaries

def get_analysis(tweet_text: str, tweet_id: str, analysis_cache: Dict[str, Dict[str, str]], model: Optional[str], provider: Optional[str]) -> str:
    if not provider:
        logger.error("Provider is not specified for tweet analysis.")
        return "Analysis provider not specified."

    if provider not in ['openai', 'claude']:
        logger.error(f"Unsupported provider: {provider}")
        return f"Unsupported provider: {provider}"
    
    cache = load_cache()
    analysis_cache = cache.get(f'{provider}_analyses', {})

    cache_key = f"{tweet_id}_{model}_{provider}"
    if cache_key in analysis_cache:
        analysis = analysis_cache[cache_key]
        logger.info(f"Using cached {provider} analysis for tweet {tweet_id}")
        return analysis

    logger.info(f"Generating new {provider} analysis for tweet {tweet_id}")
    
    prompt = f"""Analyze the following tweet:

{tweet_text}

1. Provide a concise paraphrase of the tweet.
2. Determine if the tweet indicates a short or long position, which instruments are mentioned, and at what levels.
   Be specific about which cryptocurrency the levels refer to and include all mentioned levels.
3. If any information is not applicable or not mentioned, do not include it in the response.

Respond in the following format:
Paraphrase: [Your paraphrase here]
Trading Position: [Short/Long] (if applicable)
Instruments: [Instruments mentioned] (if applicable)
Levels: [All levels mentioned with associated cryptocurrency, maintaining the original precision] (if applicable)"""

    try:
        if provider == 'openai':
            client = OpenAI(api_key=OpenAI_API_KEY)
            response = client.chat.completions.create(
                model=model or "gpt-4o-mini",
                messages=[{"role": "user", "content": prompt}],
                temperature=0.7,
                max_tokens=256,
                top_p=1,
                frequency_penalty=0,
                presence_penalty=0
            )
            analysis = response.choices[0].message.content
        elif provider == 'claude':
            client = anthropic.Client(api_key=CLAUDE_API_KEY)
            response = client.messages.create(
                model=model or "claude-3-sonnet-20240229",
                max_tokens=300,
                messages=[{"role": "user", "content": prompt}]
            )
            analysis = response.content[0].text
        else:
            raise ValueError(f"Unsupported provider: {provider}")

        analysis = f"{provider.capitalize()} Analysis:\n{analysis}"
        analysis_cache[cache_key] = analysis
        cache[f'{provider}_analyses'] = analysis_cache
        save_cache(cache)
        logger.info(f"New {provider} analysis generated and cached for tweet {tweet_id}")
        return analysis
    except Exception as e:
        logger.exception(f"Error getting {provider} analysis for tweet {tweet_id}: {str(e)}")
        return f"Error in {provider} analysis: {str(e)}"
    
def generate_html(
    tweets_by_account: Dict[str, List[Dict[str, Any]]],
    youtube_summaries: Dict[str, List[Dict[str, Any]]],
    analysis_cache: Dict[str, Dict[str, str]],
    tweet_model: Optional[str],
    tweet_provider: Optional[str]
) -> str:
    html_content = """
    <!DOCTYPE html>
    <html lang="en">
    <head>
        <meta charset="UTF-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <title>Crypto Analysis</title>
        <script src="https://cdn.tailwindcss.com"></script>
        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css">
        <style>
            body {
                background-color: #1a1b1e;
                color: #e4e6eb;
            }
            .toggle-content {
                display: flex;
                flex-direction: column;
            }
            .original-content, .analysis-content {
                transition: order 0.3s ease;
            }
            .content-container {
                max-width: 800px;
                margin-left: 260px;
            }
            .header {
                position: fixed;
                top: 0;
                left: 260px;
                right: 0;
                background-color: #1a1b1e;
                z-index: 10;
                padding: 1rem;
            }
            .tweet, .youtube-summary {
                background-color: #242526;
                border-radius: 8px;
                margin-bottom: 1rem;
                padding: 1rem;
            }
            .analysis {
                background-color: #3a3b3c;
                border-radius: 8px;
                padding: 1rem;
                margin-top: 1rem;
            }
            .nav-sidebar {
                position: fixed;
                top: 0;
                left: 0;
                bottom: 0;
                width: 250px;
                background-color: #242526;
                padding: 1rem;
                overflow-y: auto;
            }
            .nav-item {
                cursor: pointer;
                padding: 0.5rem 1rem;
                border-radius: 0.25rem;
            }
            .nav-item:hover {
                background-color: #3a3b3c;
            }
            .nav-item.active {
                background-color: #3a3b3c;
            }
            .main-content {
                margin-top: 60px;
                padding: 1rem;
            }
            .toggle-content.analysis-first .analysis-content {
                background-color: rgba(255, 255, 255, 0.05);
                padding: 10px;
                border-radius: 5px;
            }
        </style>
        <script>
            function toggleContentOrder() {
                const toggleContents = document.querySelectorAll('.toggle-content');
                toggleContents.forEach(content => {
                    const originalContent = content.querySelector('.original-content');
                    const analysisContent = content.querySelector('.analysis-content');
                    
                    if (originalContent && analysisContent) {
                        const originalOrder = window.getComputedStyle(originalContent).order;
                        originalContent.style.order = originalOrder === '1' ? '2' : '1';
                        analysisContent.style.order = originalOrder === '1' ? '1' : '2';
                        
                        // Toggle a class to indicate when analysis is shown first
                        content.classList.toggle('analysis-first');
                    }
                });
            }
            function showContent(contentType) {
                document.getElementById('tweet-content').style.display = contentType === 'tweets' ? 'block' : 'none';
                document.getElementById('youtube-content').style.display = contentType === 'youtube' ? 'block' : 'none';
                document.querySelectorAll('.nav-item').forEach(item => {
                    item.classList.toggle('active', item.getAttribute('data-content') === contentType);
                });
            }
        </script>
    </head>
    <body class="p-4">
        <nav class="nav-sidebar">
            <h2 class="text-xl font-bold mb-4">Navigation</h2>
            <ul>
                <li class="nav-item active" data-content="tweets" onclick="showContent('tweets')">Tweets</li>
                <li class="nav-item" data-content="youtube" onclick="showContent('youtube')">YouTube</li>
            </ul>
        </nav>
        <div class="header flex justify-start items-center">
            <a href="#first-specific-position" class="px-4 py-2 bg-blue-600 hover:bg-blue-700 text-white font-bold rounded">
                First Specific Position
            </a>
            <div class="flex items-center space-x-4 ml-2">
                <button onclick="toggleContentOrder()" class="px-4 py-2 bg-gray-600 hover:bg-gray-700 text-white font-bold rounded">
                    Toggle Content Order
                </button>
                <button class="p-2 bg-gray-600 hover:bg-gray-700 rounded">
                    <i class="fas fa-moon text-white"></i>
                </button>
            </div>
        </div>
        <div class="main-content">
            <div id="tweet-content" class="content-container">
    """

    html_content += generate_tweet_content(tweets_by_account, analysis_cache, tweet_model, tweet_provider)

    html_content += """
            </div>
            <div id="youtube-content" class="content-container" style="display: none;">
    """

    html_content += generate_youtube_content(youtube_summaries)

    html_content += """
            </div>
        </div>
    </body>
    </html>
    """

    return html_content



def generate_tweet_content(
    tweets_by_account: Dict[str, List[Dict[str, Any]]],
    analysis_cache: Dict[str, Dict[str, str]],
    tweet_model: Optional[str],
    tweet_provider: Optional[str]
) -> str:
    html_content = ""
    all_tweets = []
    for display_name, tweets in tweets_by_account.items():
        for tweet in tweets:
            tweet['display_name'] = display_name
        all_tweets.extend(tweets)
    
    all_tweets.sort(key=lambda x: datetime.fromisoformat(x.get('created_at', '1970-01-01T00:00:00+00:00')), reverse=True)

    first_specific_position_found = False

    for tweet in all_tweets:
        created_at = datetime.fromisoformat(tweet.get('created_at', '1970-01-01T00:00:00+00:00')).replace(tzinfo=timezone.utc)
        formatted_date = created_at.strftime("%d %B %Y, %I:%M%p %Z")
        
        analysis = get_analysis(tweet.get('text', ''), str(tweet.get('id', '')), analysis_cache, tweet_model, tweet_provider)
        
        # Skip empty analyses
        if not analysis.strip():
            continue

        # Split the analysis into lines and filter out N/A lines
        analysis_lines = [line.strip() for line in analysis.splitlines() if "N/A" not in line.strip()]
        
        # Extract analysis elements
        paraphrase = ''
        trading_position = ''
        instruments = ''
        levels = ''
        
        # Remove the analysis heading if present
        analysis_lines = [line for line in analysis_lines if not line.startswith(f"{tweet_provider.capitalize()} Analysis:")]

        for line in analysis_lines:
            if line.startswith('Paraphrase:'):
                paraphrase = line.split(':', 1)[1].strip()
            elif line.startswith('Trading Position:'):
                trading_position = line.split(':', 1)[1].strip()
            elif line.startswith('Instruments:'):
                instruments = line.split(':', 1)[1].strip()
            elif line.startswith('Levels:'):
                levels = line.split(':', 1)[1].strip()

        image_html = ""
        if tweet.get('images'):
            image_html = '<div class="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-4 mb-4">'
            for img_url in tweet['images']:
                escaped_url = html.escape(img_url)
                image_html += f'<img src="{escaped_url}" alt="Tweet image" class="w-full h-48 object-cover rounded" onerror="this.onerror=null;this.src=\'https://via.placeholder.com/300x200?text=Image+Not+Available\';">'
            image_html += '</div>'

        first_specific_position_attr = ''
        if not first_specific_position_found and "Levels:" in levels and "Levels: None mentioned" not in levels:
            first_specific_position_attr = 'id="first-specific-position"'
            first_specific_position_found = True

        tweet_url = f"https://twitter.com/{tweet.get('twscraper_name', '')}/status/{tweet.get('id', '')}"

        analysis_sections = []
        if paraphrase and paraphrase.lower() not in ["n/a", "none mentioned"]:
            analysis_sections.append(f"""
                <div>
                    <h5 class="font-bold">Paraphrase:</h5>
                    <p>{html.escape(paraphrase).replace('\n', '<br>')}</p>
                </div>
            """)
        if trading_position and trading_position.lower() not in ["n/a", "none mentioned"]:
            analysis_sections.append(f"""
                <div>
                    <h5 class="font-bold">Trading Position:</h5>
                    <p>{html.escape(trading_position)}</p>
                </div>
            """)
        if instruments and instruments.lower() not in ["n/a", "none mentioned"]:
            analysis_sections.append(f"""
                <div>
                    <h5 class="font-bold">Instruments:</h5>
                    <p>{html.escape(instruments)}</p>
                </div>
            """)
        if levels and levels.lower() not in ["n/a", "none mentioned"]:
            analysis_sections.append(f"""
                <div>
                    <h5 class="font-bold">Levels:</h5>
                    <p>{html.escape(levels)}</p>
                </div>
            """)

        analysis_content = "".join(analysis_sections)

        html_content += f"""
            <div {first_specific_position_attr} class="tweet mb-8 p-6 rounded-lg shadow-md">
                <div class="tweet-header">
                    <h3 class="text-xl font-bold mb-2">Account: {tweet.get('display_name', 'Unknown')}<br>Date: {formatted_date}</h3>
                    <a href="{tweet_url}" target="_blank" class="text-blue-400 hover:text-blue-300">View Tweet</a>
                </div>
                <div class="toggle-content">
                    <div class="original-content" style="order: 1;">
                        <div class="tweet-content mt-4">
                            <p>{html.escape(tweet.get('text', '')).replace('\n', '<br>')}</p>
                        </div>
                        <div class="text-sm text-gray-400 mb-4">
                            Likes: {tweet.get('likes', 0)} | Retweets: {tweet.get('retweets', 0)} | Replies: {tweet.get('replies', 0)}
                        </div>
                        {image_html}
                    </div>
                    <div class="analysis-content" style="order: 2;">
                        {analysis_content}
                    </div>
                </div>
            </div>
        """

    return html_content

def generate_youtube_content(youtube_summaries: Dict[str, List[Dict[str, Any]]]) -> str:
    html_content = ""
    for channel, summaries in youtube_summaries.items():
        for summary in summaries:
            initial_summary = summary.get('initial_summary', {})
            targeted_analysis = summary.get('targeted_analysis', {})
            
            html_content += f"""
                <div class="youtube-summary mb-8 p-6 rounded-lg shadow-md">
                    <h3 class="text-xl font-bold mb-2">Channel: {channel}</h3>
                    <h4 class="text-lg font-semibold mb-2">Title: {summary.get('video_title', 'N/A')}</h4>
                    <a href="{summary.get('video_url', '#')}" target="_blank" class="text-blue-400 hover:text-blue-300">View Video</a>
                    
                    <div class="mt-4">
                        <h5 class="font-bold">Comprehensive Overview:</h5>
                        <p>{initial_summary.get('comprehensive_overview', 'N/A')}</p>
                    </div>
                    
                    <div class="mt-4">
                        <h5 class="font-bold">Key Points:</h5>
                        <ul>
                            {' '.join(f'<li>{point}</li>' for point in initial_summary.get('key_points', ['N/A']))}
                        </ul>
                    </div>
                    
                    <div class="mt-4">
                        <h5 class="font-bold">Notable Quotes:</h5>
                        <ul>
                            {' '.join(f'<li>"{quote}" - [Timestamp: {timestamp}]</li>' for quote, timestamp in initial_summary.get('notable_quotes', [('N/A', 'N/A')]))}
                        </ul>
                    </div>
                    
                    <div class="mt-4">
                        <h5 class="font-bold">Sentiment and Confidence:</h5>
                        <p>{initial_summary.get('sentiment_and_confidence', 'N/A')}</p>
                    </div>
                    
                    <div class="mt-4">
                        <h5 class="font-bold">Visual Aids:</h5>
                        <p>{initial_summary.get('visual_aids', 'N/A')}</p>
                    </div>
                    
                    <div class="mt-4">
                        <h5 class="font-bold">Cryptocurrencies Discussed:</h5>
                        <ul>
                            {' '.join(f'<li>{coin}: {details}</li>' for coin, details in initial_summary.get('cryptocurrencies_discussed', {'N/A': 'N/A'}).items())}
                        </ul>
                    </div>
                    
                    <div class="mt-4">
                        <h5 class="font-bold">Critical Evaluation:</h5>
                        <p>{targeted_analysis.get('critical_evaluation', 'N/A')}</p>
                    </div>
                    
                    <div class="mt-4">
                        <h5 class="font-bold">Market Comparison:</h5>
                        <p>{targeted_analysis.get('market_comparison', 'N/A')}</p>
                    </div>
                    
                    <div class="mt-4">
                        <h5 class="font-bold">Potential Risks:</h5>
                        <ul>
                            {' '.join(f'<li>{risk}</li>' for risk in targeted_analysis.get('potential_risks', ['N/A']))}
                        </ul>
                    </div>
                    
                    <div class="mt-4">
                        <h5 class="font-bold">Alternative Viewpoints:</h5>
                        <p>{targeted_analysis.get('alternative_viewpoints', 'N/A')}</p>
                    </div>
                    
                    <div class="mt-4">
                        <h5 class="font-bold">Technical Analysis:</h5>
                        <p>{targeted_analysis.get('technical_analysis', 'N/A')}</p>
                    </div>
                </div>
            """
    return html_content

async def main() -> None:
    logger.info("Entering main function")
    try:
        parser = argparse.ArgumentParser(
            description='Analyze crypto-related content from Twitter and YouTube.',
            epilog="""
Examples:
  python tweetreader.py --yt-model gpt-4o-mini --tweet-model claude-3-sonnet-20240229
  python tweetreader.py --tweet-model claude-3-sonnet-20240229 --yt-model gpt-4o-mini --yt-chunk-model gpt-3.5-turbo
  python tweetreader.py --purge-cache all --yt-model gpt-4o-mini --tweet-model claude-3-sonnet-20240229

Available Models:
  OpenAI: gpt-4o-mini, gpt-4, gpt-3.5-turbo
  Claude: claude-3-opus-20240229, claude-3-sonnet-20240229, claude-3-haiku-20240307, claude-2.1
            """,
            formatter_class=argparse.RawDescriptionHelpFormatter
        )
        parser.add_argument('--purge-cache', choices=['all', 'tweets', 'yt-summary', 'yt-all'], help='Purge the specified cache (deprecated)')
        parser.add_argument('--yt-model', required=True, help='Model to use for YouTube summarization')
        parser.add_argument('--tweet-model', required=True, help='Model to use for tweet analysis')
        parser.add_argument('--yt-chunk-model', help='Model to use for YouTube chunk summarization (if different from overall)')
        parser.add_argument('--yt-videos', type=int, default=1, help='Number of YouTube videos to analyze per channel')
        args = parser.parse_args()

        if args.purge_cache:
            warnings.warn("The --purge-cache option is deprecated. The script now uses model-specific caching.", DeprecationWarning)
            purge_cache(args.purge_cache)

        logger.info("Starting the main function")
        api = API()

        logger.info("Parsing Netscape cookies")
        cookies = parse_netscape_cookies(COOKIE_FILE)
        logger.info("Adding account to API pool")
        await api.pool.add_account("dummy_user", "dummy_pass", "dummy@email.com", "dummy_pass", cookies=cookies)

        # Define the accounts to fetch with display name and twscraper name
        accounts: Dict[str, str] = {
            "Astronomer Zero": "astronomer_zero",
            "Credible Crypto": "crediblecrypto",
            "0xEnas": "0xEnas"
        }

        # Define YouTube channels and videos to analyze
        youtube_sources: Dict[str, Tuple[str, str, Optional[int]]] = {
            "Technical Roundup": ("channel", "https://www.youtube.com/@TechnicalRoundup", 1),
            "CBS - 240824": ("video", "https://www.youtube.com/watch?v=goYZF12HYlo", None)
        }

        tweets_by_account: Dict[str, List[Dict[str, Any]]] = {}
        analysis_cache: Dict[str, Dict[str, str]] = {}  # Initialize a shared analysis cache

        # Validate and retrieve providers
        yt_model, yt_provider = validate_model_provider(args.yt_model, None)
        tweet_model, tweet_provider = validate_model_provider(args.tweet_model, None)
        yt_chunk_model, yt_chunk_provider = validate_model_provider(args.yt_chunk_model, None)

        # Sanity checks
        if not yt_provider:
            logger.error(f"Unable to determine provider for YouTube model '{yt_model}'. Please check the model name.")
            sys.exit(1)
        if not tweet_provider:
            logger.error(f"Unable to determine provider for Tweet model '{tweet_model}'. Please check the model name.")
            sys.exit(1)

        for display_name, twscraper_name in accounts.items():
            logger.info(f"Fetching tweets for user: {display_name} ({twscraper_name})")
            try:
                tweets, account_analysis_cache = await fetch_and_cache_tweets(api, twscraper_name)
                if tweets is not None:
                    for tweet in tweets:
                        tweet['display_name'] = display_name
                        tweet['twscraper_name'] = twscraper_name
                    tweets_by_account[display_name] = tweets
                    analysis_cache.update(account_analysis_cache)  # Update the shared analysis cache
                else:
                    logger.error(f"Failed to fetch tweets for {display_name} ({twscraper_name})")
            except Exception as e:
                logger.exception(f"Error processing tweets for {display_name} ({twscraper_name}): {str(e)}")

        if not any(tweets_by_account.values()):
            logger.warning(f"No tweets available for any of the specified accounts. Exiting.")
            return

        # Fetch and analyze YouTube content
        youtube_summaries: Dict[str, List[Dict[str, Any]]] = {}
        for source_name, source_info in youtube_sources.items():
            logger.info(f"Processing YouTube content for: {source_name}")
            try:
                if source_info[0] == "channel":
                    channel_url = source_info[1]
                    subtitles = fetch_youtube_subtitles(channel_url, args.yt_videos, model=yt_model)
                elif source_info[0] == "video":
                    video_url = source_info[1]
                    subtitles = fetch_youtube_subtitles(video_url, 1, model=yt_model)
                else:
                    logger.error(f"Invalid source type for {source_name}")
                    continue

                if subtitles:
                    logger.info(f"Fetched {len(subtitles)} subtitles for {source_name}")
                    summaries = analyze_youtube_subtitles(
                        subtitles,
                        yt_model, yt_provider,
                        yt_chunk_model, yt_chunk_provider
                    )
                    youtube_summaries[source_name] = summaries
                else:
                    logger.warning(f"No subtitles found for: {source_name}")
            except Exception as e:
                logger.exception(f"Error processing YouTube content for {source_name}: {str(e)}")

        logger.info(f"Generating HTML for {len(tweets_by_account)} Twitter accounts and {len(youtube_summaries)} YouTube sources")
        html_output = generate_html(tweets_by_account, youtube_summaries, analysis_cache, tweet_model, tweet_provider)

        output_file = 'crypto_analysis.html'
        logger.info(f"Writing output to {output_file}")
        with open(output_file, 'w', encoding='utf-8') as f:
            f.write(html_output)

        logger.info(f"Analysis complete. Results saved in {output_file}")
    except Exception as e:
        logger.exception(f"An error occurred in the main function: {str(e)}")
    logger.info("Exiting main function")

if __name__ == "__main__":
    logger.info("Script started")
    asyncio.run(main())
    logger.info("Script finished")

TESTS
import unittest
import os
from unittest.mock import patch, MagicMock
from tweetreader import fetch_youtube_subtitles, convert_vtt_to_text, YOUTUBE_CACHE_DIR

class TestYoutubeSubtitles(unittest.TestCase):

    def setUp(self):
        self.test_video_url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" 
        self.test_video_id = "dQw4w9WgXcQ"
        self.test_cache_dir = YOUTUBE_CACHE_DIR
        self.test_model = "gpt-4o-mini"

    def tearDown(self):
        # Clean up any downloaded files after each test
        subtitle_file = os.path.join(self.test_cache_dir, f"{self.test_video_id}.en.vtt")
        if os.path.exists(subtitle_file):
            os.remove(subtitle_file)

    @patch('tweetreader.YoutubeDL')  
    def test_fetch_youtube_subtitles_single_video(self, mock_YoutubeDL):
        # Create a mock instance of YoutubeDL
        mock_ydl_instance = mock_YoutubeDL.return_value

        # Correctly mock the context manager methods
        mock_ydl_instance.enter.return_value = mock_ydl_instance
        mock_ydl_instance.exit.return_value = None

        # Prepare the mock info returned by extract_info
        mock_info = {
            'id': self.test_video_id,
            'title': "Never Gonna Give You Up",
            'url': self.test_video_url,
            # Include other fields that your function uses
            'subtitles': {},  # Adjust based on your function's requirements
            'requested_subtitles': {},
        }

        # Ensure extract_info returns the actual dictionary
        mock_ydl_instance.extract_info.return_value = mock_info

        # Call the function with a single video URL
        subtitles = fetch_youtube_subtitles(
            self.test_video_url,
            num_videos=1,
            cache_dir=self.test_cache_dir,
            model=self.test_model
        )

        # Assertions
        self.assertEqual(len(subtitles), 1)
        self.assertEqual(subtitles[0]['video_id'], self.test_video_id)
        self.assertEqual(subtitles[0]['title'], "Never Gonna Give You Up")
        self.assertEqual(subtitles[0]['url'], self.test_video_url)
        self.assertIn('subtitles', subtitles[0])
        self.assertEqual(subtitles[0]['subtitles'], "")  # Adjust as needed        
    def test_convert_vtt_to_text_valid_file(self):
        # Create a sample VTT file for testing
        vtt_content = """WEBVTT

1
00:00:01.000 --> 00:00:05.000
This is the first line.

2
00:00:06.000 --> 00:00:10.000
This is the second line."""
        vtt_file_path = os.path.join(self.test_cache_dir, "test_subtitles.vtt")
        with open(vtt_file_path, 'w') as f:
            f.write(vtt_content)

        # Call the function and assert the output
        text = convert_vtt_to_text(vtt_file_path)
        expected_text = "This is the first line.\nThis is the second line."
        self.assertEqual(text, expected_text)

        # Clean up the test file
        os.remove(vtt_file_path)

    def test_convert_vtt_to_text_empty_file(self):
        # Create an empty VTT file for testing
        vtt_file_path = os.path.join(self.test_cache_dir, "test_subtitles_empty.vtt")
        with open(vtt_file_path, 'w') as f:
            f.write("")

        # Call the function and assert the output
        text = convert_vtt_to_text(vtt_file_path)
        self.assertEqual(text, "")

        # Clean up the test file
        os.remove(vtt_file_path)

    def test_convert_vtt_to_text_file_not_found(self):
        # Call the function with a non-existent file path
        text = convert_vtt_to_text("non_existent_file.vtt")
        self.assertEqual(text, "")

if __name__ == "__main__":
    unittest.main()

OUTPUT
2024-09-19 19:12:51,779 - INFO - Script started
2024-09-19 19:12:51,879 - INFO - API clients initialized
.2024-09-19 19:12:51,882 - ERROR - VTT file not found: non_existent_file.vtt
..E
======================================================================
ERROR: test_fetch_youtube_subtitles_single_video (test_tweetreader.TestYoutubeSubtitles.test_fetch_youtube_subtitles_single_video)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/opt/homebrew/Cellar/python@3.12/3.12.5/Frameworks/Python.framework/Versions/3.12/lib/python3.12/unittest/mock.py", line 1390, in patched
    with self.decoration_helper(patched,
  File "/opt/homebrew/Cellar/python@3.12/3.12.5/Frameworks/Python.framework/Versions/3.12/lib/python3.12/contextlib.py", line 137, in __enter__
    return next(self.gen)
           ^^^^^^^^^^^^^^
  File "/opt/homebrew/Cellar/python@3.12/3.12.5/Frameworks/Python.framework/Versions/3.12/lib/python3.12/unittest/mock.py", line 1372, in decoration_helper
    arg = exit_stack.enter_context(patching)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/Cellar/python@3.12/3.12.5/Frameworks/Python.framework/Versions/3.12/lib/python3.12/contextlib.py", line 526, in enter_context
    result = _enter(cm)
             ^^^^^^^^^^
  File "/opt/homebrew/Cellar/python@3.12/3.12.5/Frameworks/Python.framework/Versions/3.12/lib/python3.12/unittest/mock.py", line 1461, in __enter__
    original, local = self.get_original()
                      ^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/Cellar/python@3.12/3.12.5/Frameworks/Python.framework/Versions/3.12/lib/python3.12/unittest/mock.py", line 1434, in get_original
    raise AttributeError(
AttributeError: <module 'tweetreader' from '/Users/mcb/Library/Mobile Documents/com~apple~CloudDocs/Code/Python/crypto/twitter/tweetreader.py'> does not have the attribute 'YoutubeDL'

----------------------------------------------------------------------
Ran 4 tests in 0.005s

FAILED (errors=1)