Can you compare the functionalities between the two code below sentiment.py newsent.py, and tell me wic one would be better and the one that would produce accurate results

#sentiment.py
import requests
from bs4 import BeautifulSoup
from textblob import TextBlob
import pandas as pd
import time
from newspaper import Article
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import concurrent.futures
import logging
import json
import os
from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, func, Text
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
import numpy as np
from datetime import datetime, timedelta
from urllib.parse import urlparse
import re
from tenacity import retry, stop_after_attempt, wait_exponential
from statsmodels.tsa.seasonal import seasonal_decompose
from functools import lru_cache
from ratelimit import limits, sleep_and_retry

# Additional imports for new features
import tweepy  # For Twitter API
import praw    # For Reddit API
from telethon import TelegramClient
import nltk
from nltk.corpus import stopwords  # For NLP preprocessing
from nltk.tokenize import word_tokenize
import gensim
from gensim.summarization.summarizer import summarize  # For NLG
import spacy  # For NLU
import webbrowser

# Ensure NLTK resources are downloaded
nltk.download('stopwords')
nltk.download('punkt')

# Load spacy model
nlp = spacy.load('en_core_web_sm')

Base = declarative_base()

class SentimentData(Base):
    __tablename__ = 'sentiment_data'
    id = Column(Integer, primary_key=True)
    url = Column(String, unique=True)
    sentiment = Column(Float)
    timestamp = Column(DateTime)
    source = Column(String)  # Added source field to indicate the data source

class CacheData(Base):
    __tablename__ = 'cache_data'
    url = Column(String, primary_key=True)
    content = Column(Text)
    timestamp = Column(DateTime)

class TradingSignal(Base):
    __tablename__ = 'trading_signals'
    id = Column(Integer, primary_key=True)
    signal = Column(String)
    confidence = Column(Float)
    timestamp = Column(DateTime)

class SentimentAnalyzer:
    def __init__(self, config_path='config.json'):
        self.load_config(config_path)
        self.vader_analyzer = SentimentIntensityAnalyzer()
        self.session = requests.Session()
        self.session.headers.update({'User-Agent': 'Mozilla/5.0'})
        logging.basicConfig(level=logging.INFO, filename='sentiment_analysis.log')
        self.db_engine = create_engine(self.config['database_url'])
        Base.metadata.create_all(self.db_engine)
        self.db_session = sessionmaker(bind=self.db_engine)()
        self.domain_last_request = {}

        # Initialize Twitter API
        self.twitter_api = self.init_twitter_api()
        # Initialize Reddit API
        self.reddit_api = self.init_reddit_api()
        # Initialize Telegram Client
        self.telegram_client = self.init_telegram_client()
        # Initialize stopwords
        self.stop_words = set(stopwords.words('english'))

    def load_config(self, config_path):
        try:
            with open(config_path, 'r') as f:
                self.config = json.load(f)
        except FileNotFoundError:
            logging.error(f"Config file not found: {config_path}")
            raise
        except json.JSONDecodeError:
            logging.error(f"Invalid JSON in config file: {config_path}")
            raise

    # Initialize Twitter API
    def init_twitter_api(self):
        try:
            auth = tweepy.OAuth1UserHandler(
                self.config['twitter_api_key'],
                self.config['twitter_api_secret'],
                self.config['twitter_access_token'],
                self.config['twitter_access_token_secret']
            )
            api = tweepy.API(auth)
            return api
        except Exception as e:
            logging.error(f"Error initializing Twitter API: {str(e)}")
            return None

    # Initialize Reddit API
    def init_reddit_api(self):
        try:
            reddit = praw.Reddit(
                client_id=self.config['reddit_client_id'],
                client_secret=self.config['reddit_client_secret'],
                user_agent=self.config['reddit_user_agent']
            )
            return reddit
        except Exception as e:
            logging.error(f"Error initializing Reddit API: {str(e)}")
            return None

    # Initialize Telegram Client
    def init_telegram_client(self):
        try:
            client = TelegramClient(
                self.config['telegram_session_name'],
                self.config['telegram_api_id'],
                self.config['telegram_api_hash']
            )
            client.start()
            return client
        except Exception as e:
            logging.error(f"Error initializing Telegram Client: {str(e)}")
            return None

    @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
    @sleep_and_retry
    @limits(calls=1, period=1)  # Limit to 1 call per second
    def fetch_news_api(self, query="cryptocurrency", page_size=5):
        url = f"https://newsapi.org/v2/everything?q={query}&apiKey={self.config['news_api_key']}&pageSize={page_size}"
        try:
            response = self.session.get(url, timeout=10)
            response.raise_for_status()
            return response.json()['articles']
        except requests.RequestException as e:
            logging.error(f"News API request failed: {str(e)}")
            raise

    # Fetch articles from additional news sources
    def fetch_additional_news(self, query="cryptocurrency"):
        articles = []
        for source in self.config.get('additional_news_sources', []):
            try:
                response = self.session.get(source.format(query=query), timeout=10)
                response.raise_for_status()
                soup = BeautifulSoup(response.text, 'html.parser')
                links = [a['href'] for a in soup.find_all('a', href=True) if query.lower() in a.text.lower()]
                articles.extend(links)
            except Exception as e:
                logging.error(f"Error fetching from source {source}: {str(e)}")
        return articles

    @lru_cache(maxsize=100)
    def scrape_article(self, url):
        domain = urlparse(url).netloc
        self.rate_limit(domain)
        
        # Check cache first
        cached_content = self.get_from_cache(url)
        if cached_content:
            return cached_content

        try:
            article = Article(url)
            article.download()
            article.parse()
            content = article.text
            self.add_to_cache(url, content)
            return content
        except Exception as e:
            logging.error(f"Error scraping {url}: {str(e)}")
            return ""

    def get_from_cache(self, url):
        cache_data = self.db_session.query(CacheData).filter(
            CacheData.url == url,
            CacheData.timestamp > datetime.now() - timedelta(days=1)
        ).first()
        return cache_data.content if cache_data else None

    def add_to_cache(self, url, content):
        cache_data = CacheData(url=url, content=content, timestamp=datetime.now())
        self.db_session.merge(cache_data)
        self.db_session.commit()

    def clear_cache(self):
        self.scrape_article.cache_clear()  # Clear the LRU cache
        # Clear old entries from the database cache
        self.db_session.query(CacheData).filter(
            CacheData.timestamp <= datetime.now() - timedelta(days=7)
        ).delete()
        self.db_session.commit()

    def rate_limit(self, domain):
        current_time = time.time()
        if domain in self.domain_last_request:
            elapsed = current_time - self.domain_last_request[domain]
            if elapsed < self.config['per_domain_rate_limit']:
                time.sleep(self.config['per_domain_rate_limit'] - elapsed)
        self.domain_last_request[domain] = time.time()

    def preprocess_text(self, text):
        # Additional preprocessing steps
        text = re.sub(r'http\S+', '', text)  # Remove URLs
        text = re.sub(r'[^a-zA-Z\s]', '', text)  # Remove non-alphabetic characters
        text = text.lower()
        # Remove stopwords
        words = word_tokenize(text)
        filtered_words = [word for word in words if word not in self.stop_words]
        return ' '.join(filtered_words)

    def get_sentiment(self, text):
        try:
            preprocessed_text = self.preprocess_text(text)
            blob_sentiment = TextBlob(preprocessed_text).sentiment.polarity
            vader_sentiment = self.vader_analyzer.polarity_scores(preprocessed_text)['compound']
            return self.config['textblob_weight'] * blob_sentiment + self.config['vader_weight'] * vader_sentiment
        except Exception as e:
            logging.error(f"Error in sentiment analysis: {str(e)}")
            return 0

    def analyze_text_nlu(self, text):
        doc = nlp(text)
        entities = [(ent.text, ent.label_) for ent in doc.ents]
        return entities

    def process_url(self, url):
        text = self.scrape_article(url)
        sentiment = self.get_sentiment(text)
        insights = self.analyze_text_nlu(text)
        return {
            'url': url, 
            'sentiment': sentiment, 
            'timestamp': datetime.now(), 
            'source': 'News',
            'insights': insights
        }

    # New method to fetch and process Twitter data
    def fetch_twitter_data(self, query, count=100):
        tweets = []
        if self.twitter_api:
            try:
                for tweet in tweepy.Cursor(self.twitter_api.search_tweets, q=query, lang='en').items(count):
                    tweets.append(tweet.text)
                return tweets
            except Exception as e:
                logging.error(f"Error fetching tweets: {str(e)}")
        return tweets

    # New method to fetch and process Reddit data
    def fetch_reddit_data(self, subreddit_name='CryptoCurrency', limit=100):
        posts = []
        if self.reddit_api:
            try:
                subreddit = self.reddit_api.subreddit(subreddit_name)
                for post in subreddit.hot(limit=limit):
                    posts.append(post.title + ' ' + post.selftext)
            except Exception as e:
                logging.error(f"Error fetching Reddit posts: {str(e)}")
        return posts

    # New method to fetch and process Telegram data (public channels and groups)
    def fetch_telegram_data(self, query='cryptocurrency', limit=100):
        messages = []
        if self.telegram_client:
            try:
                async def get_messages():
                    async for dialog in self.telegram_client.iter_dialogs():
                        if dialog.is_channel or dialog.is_group:
                            async for message in self.telegram_client.iter_messages(dialog, search=query, limit=limit):
                                if message.message:
                                    messages.append(message.message)
                self.telegram_client.loop.run_until_complete(get_messages())
            except Exception as e:
                logging.error(f"Error fetching Telegram messages: {str(e)}")
        return messages

    # New method to perform data quality analysis
    def analyze_data_quality(self, data):
        quality_report = {}
        total_records = len(data)
        if total_records == 0:
            return {"error": "No data to analyze"}
        null_values = data.isnull().sum().sum()
        duplicate_records = data.duplicated().sum()
        completeness = ((total_records - null_values) / total_records) * 100
        uniqueness = ((total_records - duplicate_records) / total_records) * 100
        quality_report['total_records'] = total_records
        quality_report['null_values'] = null_values
        quality_report['duplicate_records'] = duplicate_records
        quality_report['completeness'] = completeness
        quality_report['uniqueness'] = uniqueness
        return quality_report

    # NLG method to generate detailed reports
    def generate_report(self, trend_report):
        try:
            # Summarize the trend report using NLG
            summary = summarize(trend_report, ratio=0.5)
            return summary
        except Exception as e:
            logging.error(f"Error generating report: {str(e)}")
            return trend_report

    def analyze_social_media_sentiment(self, query='cryptocurrency'):
        # Fetch data from Twitter
        twitter_data = self.fetch_twitter_data(query, count=100)
        twitter_sentiments = []
        for text in twitter_data:
            sentiment = self.get_sentiment(text)
            insights = self.analyze_text_nlu(text)
            twitter_sentiments.append({'text': text, 'sentiment': sentiment, 'timestamp': datetime.now(), 'source': 'Twitter', 'insights': insights})

        # Fetch data from Reddit
        reddit_data = self.fetch_reddit_data(limit=100)
        reddit_sentiments = []
        for text in reddit_data:
            sentiment = self.get_sentiment(text)
            insights = self.analyze_text_nlu(text)
            reddit_sentiments.append({'text': text, 'sentiment': sentiment, 'timestamp': datetime.now(), 'source': 'Reddit', 'insights': insights})

        # Fetch data from Telegram (public channels and groups)
        telegram_data = self.fetch_telegram_data(query=query, limit=100)
        telegram_sentiments = []
        for text in telegram_data:
            sentiment = self.get_sentiment(text)
            insights = self.analyze_text_nlu(text)
            telegram_sentiments.append({'text': text, 'sentiment': sentiment, 'timestamp': datetime.now(), 'source': 'Telegram', 'insights': insights})

        # Combine all sentiments
        all_social_sentiments = twitter_sentiments + reddit_sentiments + telegram_sentiments
        return pd.DataFrame(all_social_sentiments)

    def add_sentiment_features(self, data=None, query="cryptocurrency", page_size=5):
        if not isinstance(query, str) or not isinstance(page_size, int) or page_size <= 0:
            raise ValueError("Invalid input parameters")

        if data is None:
            data = pd.DataFrame()

        # Fetch news articles
        try:
            api_articles = self.fetch_news_api(query, page_size)
            api_urls = [article['url'] for article in api_articles]
            # Append additional news sources
            additional_articles = self.fetch_additional_news(query)
            api_urls.extend(additional_articles)
        except Exception as e:
            logging.error(f"Failed to fetch news articles: {str(e)}")
            api_urls = []  # Fallback to empty list if API fails

        all_urls = list(set(api_urls + self.config['crypto_sites']))

        results = []
        with concurrent.futures.ThreadPoolExecutor(max_workers=self.config['max_workers']) as executor:
            future_to_url = {executor.submit(self.process_url, url): url for url in all_urls}
            for future in concurrent.futures.as_completed(future_to_url):
                url = future_to_url[future]
                try:
                    result = future.result()
                    results.append(result)
                except Exception as e:
                    logging.error(f"Error processing {url}: {str(e)}")

        sentiment_df = pd.DataFrame(results)

        # Analyze social media sentiment
        social_sentiment_df = self.analyze_social_media_sentiment(query)

        # Combine news and social media sentiment
        combined_sentiment_df = pd.concat([sentiment_df, social_sentiment_df], ignore_index=True)

        # Perform data quality analysis
        data_quality_report = self.analyze_data_quality(combined_sentiment_df)
        logging.info(f"Data Quality Report: {data_quality_report}")

        if not data.empty:
            result = pd.concat([data, combined_sentiment_df], axis=1)
        else:
            result = combined_sentiment_df

        self.save_to_database(result)
        return result

    def save_to_database(self, data):
        try:
            data_records = data.to_dict(orient='records')
            # Optimize database interaction with batch inserts
            sentiment_data_objects = [SentimentData(
                url=record.get('url', ''),
                sentiment=record['sentiment'],
                timestamp=record['timestamp'],
                source=record.get('source', 'Unknown')
            ) for record in data_records]
            self.db_session.bulk_save_objects(sentiment_data_objects)
            self.db_session.commit()
        except Exception as e:
            self.db_session.rollback()
            logging.error(f"Error saving to database: {str(e)}")

    def analyze_trends(self, data, window_size='7D'):
        avg_sentiment = data['sentiment'].mean()
        std_sentiment = data['sentiment'].std()
        sentiment_trend = "Positive" if avg_sentiment > 0 else "Negative" if avg_sentiment < 0 else "Neutral"

        # Identify outliers
        outliers = data[np.abs(data['sentiment'] - avg_sentiment) > 2 * std_sentiment]

        # Time series analysis
        data['timestamp'] = pd.to_datetime(data['timestamp'])
        data = data.set_index('timestamp')
        daily_sentiment = data.resample(window_size)['sentiment'].mean()

        # Calculate moving average
        ma = daily_sentiment.rolling(window=self.config['moving_average_window']).mean()

        # Perform seasonal decomposition
        try:
            decomposition = seasonal_decompose(daily_sentiment.dropna(), model='additive', period=7)
            trend = decomposition.trend
            seasonal = decomposition.seasonal
            residual = decomposition.resid
        except Exception as e:
            logging.error(f"Error in seasonal decomposition: {str(e)}")
            trend = seasonal = residual = pd.Series()

        # Generate trading signals
        trading_signals = self.generate_trading_signals(data)

        # Save trading signals to database
        self.save_trading_signals(trading_signals)

        trend_report = f"""
        Overall sentiment trend: {sentiment_trend}
        Average sentiment: {avg_sentiment:.2f}
        Standard deviation: {std_sentiment:.2f}
        Number of outliers: {len(outliers)}
        Data Quality Report: {self.analyze_data_quality(data)}
        Daily sentiment trend:
        {daily_sentiment.to_string()}
        Moving Average ({self.config['moving_average_window']} periods):
        {ma.to_string()}
        Trend component:
        {trend.to_string()}
        Seasonal component:
        {seasonal.to_string()}
        Residual component:
        {residual.to_string()}
        """

        # Generate detailed report using NLG
        detailed_report = self.generate_report(trend_report)

        # Generate HTML report
        self.generate_html_report(trading_signals, detailed_report)

        return detailed_report

    def generate_trading_signals(self, data):
        trading_signals = []
        recent_sentiment = data['sentiment'].tail(self.config.get('signal_window', 10))
        avg_recent_sentiment = recent_sentiment.mean()

        if avg_recent_sentiment > self.config.get('bullish_threshold', 0.1):
            signal = 'Bullish'
            confidence = avg_recent_sentiment
        elif avg_recent_sentiment < self.config.get('bearish_threshold', -0.1):
            signal = 'Bearish'
            confidence = -avg_recent_sentiment
        else:
            signal = 'Neutral'
            confidence = avg_recent_sentiment

        trading_signal = {
            'signal': signal,
            'confidence': confidence,
            'timestamp': datetime.now()
        }
        trading_signals.append(trading_signal)

        return trading_signals

    def save_trading_signals(self, signals):
        try:
            signal_objects = [TradingSignal(
                signal=signal['signal'],
                confidence=signal['confidence'],
                timestamp=signal['timestamp']
            ) for signal in signals]
            self.db_session.bulk_save_objects(signal_objects)
            self.db_session.commit()
        except Exception as e:
            self.db_session.rollback()
            logging.error(f"Error saving trading signals to database: {str(e)}")

    def generate_html_report(self, signals, detailed_report):
        # Load HTML template
        with open('sentiment_results.html', 'r', encoding='utf-8') as f:
            html_template = f.read()

        # Generate signal tables
        bullish_table = self.generate_signal_table('Bullish')
        bearish_table = self.generate_signal_table('Bearish')
        neutral_table = self.generate_signal_table('Neutral')

        # Replace placeholders in the template
        html_content = html_template.format(
            detailed_report=detailed_report,
            bullish_signals=bullish_table,
            bearish_signals=bearish_table,
            neutral_signals=neutral_table
        )

        # Save the populated HTML
        with open('sentiment_analysis_report.html', 'w', encoding='utf-8') as f:
            f.write(html_content)

        # Optionally open the report in a web browser
        # webbrowser.open('sentiment_analysis_report.html')

    def generate_signal_table(self, signal_type):
        # Fetch signals from the database
        signals = self.db_session.query(TradingSignal).filter(TradingSignal.signal == signal_type).all()
        if not signals:
            return f"<p>No {signal_type} signals generated.</p>"
        table_html = "<table border='1' cellpadding='5' cellspacing='0'><tr><th>Signal</th><th>Confidence</th><th>Timestamp</th></tr>"
        for s in signals:
            table_html += f"<tr><td>{s.signal}</td><td>{s.confidence:.2f}</td><td>{s.timestamp.strftime('%Y-%m-%d %H:%M:%S')}</td></tr>"
        table_html += "</table>"
        return table_html

# Usage
if __name__ == "__main__":
    analyzer = SentimentAnalyzer()
    analyzer.clear_cache()  # Clear cache before starting
    result_data = analyzer.add_sentiment_features(query="bitcoin", page_size=10)
    trend_analysis = analyzer.analyze_trends(result_data)
    print(trend_analysis)

newsent.py
import requests
from bs4 import BeautifulSoup
from textblob import TextBlob
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import pandas as pd
import nltk
import spacy
from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, func, Text
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
import logging
import re
from concurrent.futures import ThreadPoolExecutor
import datetime

# Download NLTK resources
nltk.download('stopwords')
nltk.download('punkt')

# Load spaCy model
nlp = spacy.load('en_core_web_sm')

# SQLAlchemy setup
Base = declarative_base()
engine = create_engine('postgresql://username:password@localhost:5432/crypto_sentiment')
Session = sessionmaker(bind=engine)
session = Session()

# ORM Models
class SentimentData(Base):
    __tablename__ = 'sentiment_data'
    id = Column(Integer, primary_key=True)
    source = Column(String)
    sentiment = Column(Float)
    timestamp = Column(DateTime)
    crypto_symbol = Column(String)

class TradingSignal(Base):
    __tablename__ = 'trading_signals'
    id = Column(Integer, primary_key=True)
    signal = Column(String)
    confidence = Column(Float)
    timestamp = Column(DateTime)
    crypto_symbol = Column(String)

Base.metadata.create_all(engine)

# Initialize sentiment analyzers
sid = SentimentIntensityAnalyzer()

def scrape_news(url):
    try:
        response = requests.get(url)
        response.raise_for_status()
        soup = BeautifulSoup(response.content, 'html.parser')
        paragraphs = [p.get_text() for p in soup.find_all('p')]
        return ' '.join(paragraphs)
    except requests.exceptions.RequestException as e:
        logging.error(f"Error fetching URL {url}: {e}")
        return ""

def analyze_text(text):
    # Tokenization and clean-up
    text = re.sub(r'http\S+', '', text)
    text = re.sub(r'[^a-zA-Z\s]', '', text)
    text = text.lower()
    words = nltk.word_tokenize(text)

    # Sentiment Analysis
    blob_sentiment = TextBlob(text).sentiment.polarity
    vader_sentiment = sid.polarity_scores(text)['compound']

    # Calculate final sentiment score
    sentiment_score = 0.5 * blob_sentiment + 0.5 * vader_sentiment

    # NER
    doc = nlp(text)
    entities = [(ent.text, ent.label_) for ent in doc.ents]
    logging.info(f"Identified entities: {entities}")

    return sentiment_score

def generate_trading_signal(sentiment_score):
    if sentiment_score > 0.5:
        return "BUY"
    elif sentiment_score < -0.5:
        return "SELL"
    return "NEUTRAL"

def save_sentiment_data(source, sentiment, crypto_symbol):
    sentiment_data = SentimentData(
        source=source,
        sentiment=sentiment,
        timestamp=datetime.datetime.now(),
        crypto_symbol=crypto_symbol
    )
    session.add(sentiment_data)
    session.commit()

def save_trading_signal(signal, confidence, crypto_symbol):
    trading_signal = TradingSignal(
        signal=signal,
        confidence=confidence,
        timestamp=datetime.datetime.now(),
        crypto_symbol=crypto_symbol
    )
    session.add(trading_signal)
    session.commit()

def process_news_articles(crypto_symbol):
    rss_feeds = [
        f"https://news.google.com/rss/search?q={crypto_symbol}",
        "https://www.coindesk.com/arc/outboundfeeds/rss/"
    ]
    for url in rss_feeds:
        news_text = scrape_news(url)
        sentiment_score = analyze_text(news_text)
        save_sentiment_data('News', sentiment_score, crypto_symbol)
        signal = generate_trading_signal(sentiment_score)
        save_trading_signal(signal, sentiment_score, crypto_symbol)

def process_social_media(crypto_symbol):
    # Mocking the scraping function for demonstration purposes
    # Replace with actual scraping logic for Twitter and Reddit using BeautifulSoup
    social_media_text = f"Latest discussions about {crypto_symbol}"
    sentiment_score = analyze_text(social_media_text)
    save_sentiment_data('Social Media', sentiment_score, crypto_symbol)

def main():
    # Placeholder to fetch crypto symbols from fetch_data.py
    # Replace with actual DataFrame read logic
    crypto_symbols = ['BTCUSDT', 'ETHUSDT', 'XRPUSDT']  # Example symbols

    with ThreadPoolExecutor(max_workers=5) as executor:
        for crypto_symbol in crypto_symbols:
            executor.submit(process_news_articles, crypto_symbol)
            executor.submit(process_social_media, crypto_symbol)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    main()