import requests
from bs4 import BeautifulSoup
from textblob import TextBlob
import pandas as pd
import time
from newspaper import Article
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import concurrent.futures
import logging
import json
import os
from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, func, Text
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
import numpy as np
from datetime import datetime, timedelta
from urllib.parse import urlparse
import re
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
from statsmodels.tsa.seasonal import seasonal_decompose
from functools import lru_cache
from ratelimit import limits, sleep_and_retry

Base = declarative_base()

class SentimentData(Base):
    __tablename__ = 'sentiment_data'
    id = Column(Integer, primary_key=True)
    url = Column(String, unique=True)
    sentiment = Column(Float)
    timestamp = Column(DateTime)

class CacheData(Base):
    __tablename__ = 'cache_data'
    url = Column(String, primary_key=True)
    content = Column(Text)
    timestamp = Column(DateTime)

class CryptoSentimentAnalyzer:
    def __init__(self, config_path='config.json'):
        self.load_config(config_path)
        self.vader_analyzer = SentimentIntensityAnalyzer()
        self.session = requests.Session()
        self.session.headers.update({'User-Agent': 'Mozilla/5.0'})
        logging.basicConfig(level=logging.INFO, filename='sentiment_analysis.log')
        self.db_engine = create_engine(self.config['database_url'])
        Base.metadata.create_all(self.db_engine)
        self.db_session = sessionmaker(bind=self.db_engine)()
        self.domain_last_request = {}

    def load_config(self, config_path):
        try:
            with open(config_path, 'r') as f:
                self.config = json.load(f)
        except FileNotFoundError:
            logging.error(f"Config file not found: {config_path}")
            raise
        except json.JSONDecodeError:
            logging.error(f"Invalid JSON in config file: {config_path}")
            raise

    @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
    @sleep_and_retry
    @limits(calls=1, period=1)  # Limit to 1 call per second
    def fetch_news_api(self, query="cryptocurrency", page_size=5):
        url = f"https://newsapi.org/v2/everything?q={query}&apiKey={self.config['news_api_key']}&pageSize={page_size}"
        try:
            response = self.session.get(url, timeout=10)
            response.raise_for_status()
            return response.json()['articles']
        except requests.RequestException as e:
            logging.error(f"News API request failed: {str(e)}")
            raise

    @lru_cache(maxsize=100)
    def scrape_article(self, url):
        domain = urlparse(url).netloc
        self.rate_limit(domain)
        
        # Check cache first
        cached_content = self.get_from_cache(url)
        if cached_content:
            return cached_content

        try:
            article = Article(url)
            article.download()
            article.parse()
            content = article.text
            self.add_to_cache(url, content)
            return content
        except Exception as e:
            logging.error(f"Error scraping {url}: {str(e)}")
            return ""

    def get_from_cache(self, url):
        cache_data = self.db_session.query(CacheData).filter(
            CacheData.url == url,
            CacheData.timestamp > datetime.now() - timedelta(days=1)
        ).first()
        return cache_data.content if cache_data else None

    def add_to_cache(self, url, content):
        cache_data = CacheData(url=url, content=content, timestamp=datetime.now())
        self.db_session.merge(cache_data)
        self.db_session.commit()

    def clear_cache(self):
        self.scrape_article.cache_clear()  # Clear the LRU cache
        # Clear old entries from the database cache
        self.db_session.query(CacheData).filter(
            CacheData.timestamp <= datetime.now() - timedelta(days=7)
        ).delete()
        self.db_session.commit()

    def rate_limit(self, domain):
        current_time = time.time()
        if domain in self.domain_last_request:
            elapsed = current_time - self.domain_last_request[domain]
            if elapsed < self.config['per_domain_rate_limit']:
                time.sleep(self.config['per_domain_rate_limit'] - elapsed)
        self.domain_last_request[domain] = time.time()

    def preprocess_text(self, text):
        text = re.sub(r'http\S+', '', text)
        text = re.sub(r'[^a-zA-Z\s]', '', text)
        return text.lower()

    def get_sentiment(self, text):
        try:
            preprocessed_text = self.preprocess_text(text)
            blob_sentiment = TextBlob(preprocessed_text).sentiment.polarity
            vader_sentiment = self.vader_analyzer.polarity_scores(preprocessed_text)['compound']
            return self.config['textblob_weight'] * blob_sentiment + self.config['vader_weight'] * vader_sentiment
        except Exception as e:
            logging.error(f"Error in sentiment analysis: {str(e)}")
            return 0

    def process_url(self, url):
        text = self.scrape_article(url)
        sentiment = self.get_sentiment(text)
        return {'url': url, 'sentiment': sentiment, 'timestamp': datetime.now()}

    def add_sentiment_features(self, data=None, query="cryptocurrency", page_size=5):
        if not isinstance(query, str) or not isinstance(page_size, int) or page_size <= 0:
            raise ValueError("Invalid input parameters")

        if data is None:
            data = pd.DataFrame()

        try:
            api_articles = self.fetch_news_api(query, page_size)
            api_urls = [article['url'] for article in api_articles]
        except Exception as e:
            logging.error(f"Failed to fetch from News API: {str(e)}")
            api_urls = []  # Fallback to empty list if API fails

        all_urls = list(set(api_urls + self.config['crypto_sites']))

        results = []
        with concurrent.futures.ThreadPoolExecutor(max_workers=self.config['max_workers']) as executor:
            future_to_url = {executor.submit(self.process_url, url): url for url in all_urls}
            for future in concurrent.futures.as_completed(future_to_url):
                url = future_to_url[future]
                try:
                    result = future.result()
                    results.append(result)
                except Exception as e:
                    logging.error(f"Error processing {url}: {str(e)}")

        sentiment_df = pd.DataFrame(results)
        
        if not data.empty:
            result = pd.concat([data, sentiment_df], axis=1)
        else:
            result = sentiment_df

        self.save_to_database(result)
        return result

    def save_to_database(self, data):
        try:
            for _, row in data.iterrows():
                sentiment_data = SentimentData(
                    url=row['url'],
                    sentiment=row['sentiment'],
                    timestamp=row['timestamp']
                )
                self.db_session.merge(sentiment_data)  # Use merge instead of add to update existing entries
            self.db_session.commit()
        except Exception as e:
            self.db_session.rollback()
            logging.error(f"Error saving to database: {str(e)}")

    def analyze_trends(self, data, window_size='7D'):
        avg_sentiment = data['sentiment'].mean()
        std_sentiment = data['sentiment'].std()
        sentiment_trend = "Positive" if avg_sentiment > 0 else "Negative" if avg_sentiment < 0 else "Neutral"
        
        # Identify outliers
        outliers = data[np.abs(data['sentiment'] - avg_sentiment) > 2 * std_sentiment]
        
        # Time series analysis
        data['timestamp'] = pd.to_datetime(data['timestamp'])
        daily_sentiment = data.resample(window_size, on='timestamp')['sentiment'].mean()
        
        # Calculate moving average
        ma = daily_sentiment.rolling(window=self.config['moving_average_window']).mean()
        
        # Perform seasonal decomposition
        try:
            decomposition = seasonal_decompose(daily_sentiment, model='additive', period=7)
            trend = decomposition.trend
            seasonal = decomposition.seasonal
            residual = decomposition.resid
        except Exception as e:
            logging.error(f"Error in seasonal decomposition: {str(e)}")
            trend = seasonal = residual = pd.Series()

        trend_report = f"""
        Overall sentiment trend: {sentiment_trend}
        Average sentiment: {avg_sentiment:.2f}
        Standard deviation: {std_sentiment:.2f}
        Number of outliers: {len(outliers)}
        Daily sentiment trend:
        {daily_sentiment.to_string()}
        Moving Average ({self.config['moving_average_window']} periods):
        {ma.to_string()}
        Trend component:
        {trend.to_string()}
        Seasonal component:
        {seasonal.to_string()}
        Residual component:
        {residual.to_string()}
        """
        
        return trend_report

# Usage
if __name__ == "__main__":
    analyzer = CryptoSentimentAnalyzer()
    analyzer.clear_cache()  # Clear cache before starting
    result_data = analyzer.add_sentiment_features(query="bitcoin", page_size=10)
    trend_analysis = analyzer.analyze_trends(result_data)
    print(trend_analysis)
In my code, I want you to improve it. It is said keeping a close eye on platforms like Twitter, Reddit, and Telegram for indications of market sentiment, as well as following financial news sources to detect any potential impact on the crypto market.
Another approach, called sentiment analysis, involves using Natural Language Processing (NLP) techniques to analyze online messages related to cryptocurrencies and derive insights about market sentiment. Combining this information with traditional technical analysis can provide traders with a more comprehensive view of the market and help them make informed decisions.
Additionally, tracking macroeconomic data and global events that may affect the cryptocurrency market can also be useful for predicting price movements. This includes tracking factors like global economic trends, regulatory developments, geopolitical tensions, and the performance of traditional financial markets. So can you modify and improve my code to include all the above suggestions without changing existing functions. Only change where necessary and send me complete code