#fetch_data.py
import asyncio
import json
import os
import sys
import websockets
import ccxt.async_support as ccxt
import pandas as pd
import traceback
from datetime import datetime
from asgiref.sync import sync_to_async
import django
from loguru import logger

# Initialize Django
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'binancebot.settings')
django.setup()

from .models import Kline, OrderBook

# Remove default Loguru handlers and configure new ones
logger.remove()
logger.add("fetch_data.log", rotation="20 MB", retention="7 days", level="DEBUG",
           format="{time} | {level} | {message}")  # File logging
logger.add(sys.stdout, level="INFO")  # Console logging

class DataFetcher:
    def __init__(self, symbols, timeframes=['1m', '5m', '15m', '1h']):
        self.symbols = symbols
        self.timeframes = [tf.lower() for tf in timeframes]
        self.exchange = ccxt.binanceusdm({'enableRateLimit': True})
        self.data_lock = asyncio.Lock()
        self.symbol_map = {}
        self.streams = []
        self.stream_urls = []
        self.max_streams_per_connection = 100  # Adjust based on testing
        logger.debug(f"Initialized DataFetcher with symbols: {self.symbols}, timeframes: {self.timeframes}")

    async def initialize_exchange(self):
        await self.exchange.load_markets()
        for symbol in self.symbols:
            market = self.exchange.market(symbol)
            ws_symbol = market['id'].lower()  # e.g., 'btcusdt'
            self.symbol_map[ws_symbol] = symbol

            for timeframe in self.timeframes:
                self.streams.append(f"{ws_symbol}@kline_{timeframe}")

            # Add order book streams (assuming depth=5 for this example)
            self.streams.append(f"{ws_symbol}@depth5@100ms")  # Adjust depth and update speed as needed

        # Create WebSocket URLs with limited streams per connection
        for i in range(0, len(self.streams), self.max_streams_per_connection):
            stream_subset = self.streams[i:i + self.max_streams_per_connection]
            url = f"wss://fstream.binance.com/stream?streams={'/'.join(stream_subset)}"
            self.stream_urls.append(url)

        logger.debug(f"WebSocket URLs: {self.stream_urls}")

    async def fetch_historical_data(self):
        semaphore = asyncio.Semaphore(5)  # Adjusted semaphore for optimal concurrency
        tasks = []
        for symbol in self.symbols:
            for timeframe in self.timeframes:
                tasks.append(self.fetch_symbol_historical_data(symbol, timeframe, semaphore))
        await asyncio.gather(*tasks)

    async def fetch_symbol_historical_data(self, symbol, timeframe, semaphore):
        async with semaphore:
            try:
                since = None  # Fetch all available data
                all_bars = []
                limit = 1000  # Binance allows up to 1000 bars per request

                while True:
                    bars = await self.exchange.fetch_ohlcv(symbol, timeframe=timeframe, since=since, limit=limit)
                    if not bars:
                        break
                    all_bars += bars
                    since = bars[-1][0] + 1  # Prevent fetching the last bar again
                    if len(bars) < limit:
                        break

                if all_bars:
                    df = pd.DataFrame(all_bars, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
                    df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True)
                    df['timestamp'] = df['timestamp'].dt.tz_convert('Africa/Nairobi')
                    logger.info(f"Fetched {len(df)} bars for {symbol} {timeframe}")

                    # Save to database
                    await self.save_klines(symbol, timeframe, df)

            except ccxt.NetworkError as e:
                logger.error(f"Network error while fetching data for {symbol} {timeframe}: {str(e)}")
            except ccxt.ExchangeError as e:
                logger.error(f"Exchange error while fetching data for {symbol} {timeframe}: {str(e)}")
            except Exception as e:
                logger.error(f"Unexpected error fetching data for {symbol} {timeframe}: {str(e)}\n{traceback.format_exc()}")

    @sync_to_async
    def save_klines(self, symbol, timeframe, df):
        # Validate and clean data before saving
        df.dropna(inplace=True)
        if df.empty:
            logger.warning(f"No valid data to save for {symbol} {timeframe}")
            return

        klines_to_create = []
        for _, row in df.iterrows():
            klines_to_create.append(Kline(
                symbol=symbol,
                timeframe=timeframe,
                timestamp=row['timestamp'],
                open=row['open'],
                high=row['high'],
                low=row['low'],
                close=row['close'],
                volume=row['volume']
            ))
        Kline.objects.bulk_create(klines_to_create, ignore_conflicts=True)
        logger.info(f"Saved {len(klines_to_create)} klines for {symbol} {timeframe} to the database.")

    async def handle_websocket(self, url):
        retry_count = 0
        max_retries = 5
        backoff = 1

        while retry_count < max_retries:
            try:
                async with websockets.connect(
                    url,
                    ping_interval=20,
                    ping_timeout=20,
                    close_timeout=10
                ) as websocket:
                    logger.info(f"Connected to WebSocket: {url}")
                    retry_count = 0  # Reset on successful connection

                    async for message in websocket:
                        data = json.loads(message)

                        if 'data' not in data:
                            continue

                        stream_type = data['stream'].split('@')[1]  # Extract stream type (e.g., 'kline_1m', 'depth5')

                        if stream_type.startswith('kline'):
                            await self.process_kline_data(data)
                        elif stream_type.startswith('depth'):
                            await self.process_order_book_data(data)

            except websockets.exceptions.ConnectionClosed as e:
                logger.warning(f"WebSocket connection closed: {str(e)}")
            except websockets.exceptions.InvalidStatusCode as e:
                logger.error(f"Invalid status code when connecting to WebSocket: {str(e)}")
                break
            except Exception as e:
                logger.error(f"Unexpected WebSocket error: {str(e)}\n{traceback.format_exc()}")
                retry_count += 1
                sleep_time = backoff * retry_count
                logger.info(f"Retrying WebSocket connection in {sleep_time} seconds (Attempt {retry_count}/{max_retries})")
                await asyncio.sleep(sleep_time)

    async def process_kline_data(self, data):
        kline_data = data['data']['k']
        if not kline_data['x']:
            return  # Only process closed klines

        ws_symbol = kline_data['s'].lower()
        symbol = self.symbol_map.get(ws_symbol)
        timeframe = kline_data['i']

        if timeframe not in self.timeframes:
            return

        timestamp = pd.to_datetime(kline_data['t'], unit='ms', utc=True).tz_convert('Africa/Nairobi')
        kline = Kline(
            symbol=symbol,
            timeframe=timeframe,
            timestamp=timestamp,
            open=float(kline_data['o']),
            high=float(kline_data['h']),
            low=float(kline_data['l']),
            close=float(kline_data['c']),
            volume=float(kline_data['v'])
        )

        # Save or update the kline in the database
        await self.save_kline(kline)

    async def process_order_book_data(self, data):
        order_book_data = data['data']
        ws_symbol = order_book_data['s'].lower()
        symbol = self.symbol_map.get(ws_symbol)
        timestamp = pd.to_datetime(order_book_data['T'], unit='ms', utc=True).tz_convert('Africa/Nairobi')

        # Process and save order book data
        bids = [[float(price), float(qty)] for price, qty, _ in order_book_data['b']]
        asks = [[float(price), float(qty)] for price, qty, _ in order_book_data['a']]

        order_book = OrderBook(
            symbol=symbol,
            timestamp=timestamp,
            bids=bids,
            asks=asks
        )
        await self.save_order_book(order_book)

    @sync_to_async
    def save_kline(self, kline):
        # Data validation and cleansing
        if any(val is None for val in [kline.open, kline.high, kline.low, kline.close, kline.volume]):
            logger.warning(f"Invalid data encountered for {kline.symbol} {kline.timeframe} at {kline.timestamp}")
            return

        Kline.objects.update_or_create(
            symbol=kline.symbol,
            timeframe=kline.timeframe,
            timestamp=kline.timestamp,
            defaults={
                'open': kline.open,
                'high': kline.high,
                'low': kline.low,
                'close': kline.close,
                'volume': kline.volume,
            }
        )
        logger.debug(f"Saved kline for {kline.symbol} {kline.timeframe} at {kline.timestamp}")

    @sync_to_async
    def save_order_book(self, order_book):
        # You might want to add data validation here as well
        OrderBook.objects.update_or_create(
            symbol=order_book.symbol,
            timestamp=order_book.timestamp,
            defaults={
                'bids': order_book.bids,
                'asks': order_book.asks,
            }
        )
        logger.debug(f"Saved order book for {order_book.symbol} at {order_book.timestamp}")

    async def start_websockets(self):
        tasks = []
        for url in self.stream_urls:
            tasks.append(asyncio.create_task(self.handle_websocket(url)))
        await asyncio.gather(*tasks)

    async def run(self):
        try:
            logger.info("Initializing exchange...")
            await self.initialize_exchange()
            logger.info("Fetching historical data...")
            await self.fetch_historical_data()
            logger.info("Starting WebSocket connections...")
            await self.start_websockets()
        finally:
            await self.exchange.close()
            logger.info("Exchange connection closed.")

async def main():
    exchange = ccxt.binanceusdm({'enableRateLimit': True})
    await exchange.load_markets()
    markets = exchange.markets

    # Filter for USDT perpetual futures pairs
    usdt_pairs = []
    for symbol, market in markets.items():
        if market.get('active') and market.get('contract') and market.get('type') == 'swap' and market.get('linear') and market.get('quote') == 'USDT':
            usdt_pairs.append(symbol)

    usdt_pairs = list(set(usdt_pairs))
    logger.info(f"Total Futures USDT Pairs: {len(usdt_pairs)}")

    if not usdt_pairs:
        logger.warning("No USDT pairs found after filtering. Please check the filtering criteria.")
        return

    fetcher = DataFetcher(symbols=usdt_pairs, timeframes=['1m', '5m', '15m', '1h'])
    await fetcher.run()

if __name__ == "__main__":
    if sys.platform.startswith('win'):
        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
    asyncio.run(main())

#machine_model.py
import aiohttp
import numpy as np
import pandas as pd
from sklearn.feature_selection import SelectKBest, mutual_info_regression
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import RandomizedSearchCV, train_test_split, TimeSeriesSplit
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.svm import SVR
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
import tensorflow as tf
from keras import Sequential
from keras import LSTM, Dense, Dropout, Attention, Conv1D
from keras import Adam
from keras import EarlyStopping
from statsmodels.tsa.arima.model import ARIMA
from prophet import Prophet
import talib
import asyncio
from typing import Dict, List, Tuple
import logging
from fetch_data import DataFetcher  # Assume this is appropriately implemented to fetch data
from models import TradeSignal  # Assume this is your Django model for saving signals
import json

# Logger configuration
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class MachineTradingBot:
    def __init__(self, symbols: List[str]):
        self.models = {
            'random_forest': RandomForestRegressor(random_state=42),
            'svm': SVR(),
            'xgboost': XGBRegressor(random_state=42),
            'lightgbm': LGBMRegressor(random_state=42),
            'gradient_boosting': GradientBoostingRegressor(random_state=42),
            'lstm': None,
            'hybrid_cnn_lstm': None,
            'arima': None,
            'prophet': Prophet(),
        }
        self.scaler = StandardScaler()
        self.best_params = {}
        self.feature_importances = {}
        self.model_weights = {model: 1 / len(self.models) for model in self.models}
        self.model_performance = {model: [] for model in self.models}
        self.timeframes = ['5m', '15m', '1h']
        self.symbols = symbols
        self.session = None
        self.fetcher = DataFetcher(symbols=symbols, timeframes=self.timeframes)

    async def initialize(self):
        self.session = aiohttp.ClientSession()
        await self.fetcher.initialize_exchange()

    async def fetch_data(self, symbol: str, timeframe: str) -> pd.DataFrame:
        # Fetch historical data using DataFetcher
        await self.fetcher.fetch_historical_data()
        # Placeholder for fetching real-time updates

    def preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
        data = self.add_technical_indicators(data)
        data = self.add_fourier_features(data)
        data = self.add_lagged_features(data)
        data = self.add_market_regime_features(data)
        data.dropna(inplace=True)
        return data

    def add_technical_indicators(self, data: pd.DataFrame) -> pd.DataFrame:
        for col in ['open', 'high', 'low', 'close', 'volume']:
            if col not in data.columns:
                logger.warning(f"Column {col} not found in data. Skipping technical indicators.")
                return data

        try:
            data['rsi'] = talib.RSI(data['close'])
            data['macd'], data['macd_signal'], _ = talib.MACD(data['close'])
            data['bbands_upper'], data['bbands_middle'], data['bbands_lower'] = talib.BBANDS(data['close'])
            data['adx'] = talib.ADX(data['high'], data['low'], data['close'])
            data['cci'] = talib.CCI(data['high'], data['low'], data['close'])
            data['mfi'] = talib.MFI(data['high'], data['low'], data['close'], data['volume'])
        except Exception as e:
            logger.error(f"Error calculating technical indicators: {str(e)}")
        return data

    def add_fourier_features(self, data: pd.DataFrame) -> pd.DataFrame:
        for period in [5, 10, 22]:
            for order in range(1, 3):
                data[f'fourier_sin_{period}_{order}'] = np.sin(2 * np.pi * order * data.index.dayofyear / period)
                data[f'fourier_cos_{period}_{order}'] = np.cos(2 * np.pi * order * data.index.dayofyear / period)
        return data

    def add_lagged_features(self, data: pd.DataFrame) -> pd.DataFrame:
        for lag in [1, 3, 5, 7, 14, 30]:
            data[f'close_lag_{lag}'] = data['close'].shift(lag)
            data[f'volume_lag_{lag}'] = data['volume'].shift(lag)
        return data

    def add_market_regime_features(self, data: pd.DataFrame) -> pd.DataFrame:
        returns = data['close'].pct_change()
        volatility = returns.rolling(window=22).std()
        data['market_regime'] = pd.cut(volatility, bins=3, labels=['low', 'medium', 'high'])
        data = pd.get_dummies(data, columns=['market_regime'])
        return data

    def prepare_features(self, data: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Series]:
        features = data.drop(['open', 'high', 'low', 'close', 'volume'], axis=1)
        target = data['close'].shift(-1)
        selector = SelectKBest(mutual_info_regression, k=50)
        features = pd.DataFrame(selector.fit_transform(features, target), columns=features.columns[selector.get_support()])
        return features, target

    async def train_models(self, data: pd.DataFrame):
        features, target = self.prepare_features(data)
        X_train, X_test, y_train, y_test = train_test_split(features, target, test_size=0.2, shuffle=False)
        X_train_scaled = self.scaler.fit_transform(X_train)
        X_test_scaled = self.scaler.transform(X_test)

        tasks = []
        for model_name in ['random_forest', 'svm', 'xgboost', 'lightgbm', 'gradient_boosting']:
            tasks.append(self.tune_and_train(model_name, X_train_scaled, y_train, X_test_scaled, y_test))

        tasks.append(self.train_lstm(X_train_scaled, y_train, X_test_scaled, y_test))
        tasks.append(self.train_hybrid_cnn_lstm(X_train_scaled, y_train, X_test_scaled, y_test))
        tasks.append(self.train_arima(data['close']))

        prophet_data = pd.DataFrame({'ds': data.index, 'y': data['close']})
        tasks.append(self.train_prophet(prophet_data))

        await asyncio.gather(*tasks)
        self.calculate_feature_importances(features.columns)

    async def tune_and_train(self, model_name: str, X_train: np.ndarray, y_train: pd.Series, X_test: np.ndarray, y_test: pd.Series):
        logger.info(f"Tuning and training {model_name}...")
        param_distributions = self.get_param_distributions(model_name)

        tscv = TimeSeriesSplit(n_splits=5)
        search = RandomizedSearchCV(
            self.models[model_name],
            param_distributions=param_distributions,
            n_iter=50,
            cv=tscv,
            random_state=42,
            n_jobs=-1,
            scoring='neg_mean_absolute_error'
        )
        search.fit(X_train, y_train)
        self.models[model_name] = search.best_estimator_
        self.best_params[model_name] = search.best_params_

        y_pred = self.models[model_name].predict(X_test)
        mae = mean_absolute_error(y_test, y_pred)
        self.model_performance[model_name].append(mae)

        logger.info(f"Best parameters for {model_name}: {search.best_params_}")
        logger.info(f"MAE for {model_name}: {mae}")

    async def train_lstm(self, X_train: np.ndarray, y_train: pd.Series, X_test: np.ndarray, y_test: pd.Series):
        model = Sequential([
            LSTM(100, activation='relu', input_shape=(X_train.shape[1], 1), return_sequences=True),
            Attention(),
            Dropout(0.2),
            LSTM(50, activation='relu'),
            Dropout(0.2),
            Dense(1)
        ])
        model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')

        X_train_reshaped = X_train.reshape((X_train.shape[0], X_train.shape[1], 1))
        X_test_reshaped = X_test.reshape((X_test.shape[0], X_test.shape[1], 1))

        early_stopping = EarlyStopping(patience=10, restore_best_weights=True)

        model.fit(
            X_train_reshaped, y_train,
            epochs=100,
            batch_size=32,
            validation_split=0.2,
            callbacks=[early_stopping],
            verbose=0
        )
        y_pred = model.predict(X_test_reshaped)
        mae = mean_absolute_error(y_test, y_pred)
        self.model_performance['lstm'].append(mae)
        logger.info(f"MAE for LSTM: {mae}")

        self.models['lstm'] = model

    async def train_hybrid_cnn_lstm(self, X_train: np.ndarray, y_train: pd.Series, X_test: np.ndarray, y_test: pd.Series):
        model = Sequential([
            Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(X_train.shape[1], 1)),
            Conv1D(filters=64, kernel_size=3, activation='relu'),
            LSTM(50, activation='relu', return_sequences=True),
            Attention(),
            LSTM(50, activation='relu'),
            Dense(1)
        ])
        model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')

        X_train_reshaped = X_train.reshape((X_train.shape[0], X_train.shape[1], 1))
        X_test_reshaped = X_test.reshape((X_test.shape[0], X_test.shape[1], 1))

        early_stopping = EarlyStopping(patience=10, restore_best_weights=True)

        model.fit(
            X_train_reshaped, y_train,
            epochs=100,
            batch_size=32,
            validation_split=0.2,
            callbacks=[early_stopping],
            verbose=0
        )

        y_pred = model.predict(X_test_reshaped)
        mae = mean_absolute_error(y_test, y_pred)
        self.model_performance['hybrid_cnn_lstm'].append(mae)
        logger.info(f"MAE for Hybrid CNN-LSTM: {mae}")

        self.models['hybrid_cnn_lstm'] = model

    async def train_arima(self, time_series: pd.Series):
        model = ARIMA(time_series, order=(5, 1, 0))
        fitted = model.fit()

        # Evaluate performance
        y_pred = fitted.forecast(steps=len(time_series))
        mae = mean_absolute_error(time_series, y_pred)
        self.model_performance['arima'].append(mae)
        logger.info(f"MAE for ARIMA: {mae}")

        self.models['arima'] = fitted

    async def train_prophet(self, prophet_data: pd.DataFrame):
        model = Prophet()
        model.fit(prophet_data)

        future = model.make_future_dataframe(periods=len(prophet_data))
        forecast = model.predict(future)

        y_pred = forecast['yhat'][-len(prophet_data):]
        mae = mean_absolute_error(prophet_data['y'], y_pred)
        self.model_performance['prophet'].append(mae)
        logger.info(f"MAE for Prophet: {mae}")

        self.models['prophet'] = model

    async def generate_signals(self, latest_data: pd.DataFrame) -> Tuple[Dict[str, str], Dict[str, float]]:
        features, _ = self.prepare_features(latest_data)
        scaled_features = self.scaler.transform(features)

        signals = {}
        confidences = {}

        for model_name in ['random_forest', 'svm', 'xgboost', 'lightgbm', 'gradient_boosting']:
            prediction = self.models[model_name].predict(scaled_features)
            signal_strength = prediction[-1] - latest_data['close'].iloc[-1]
            category = "strong buy" if signal_strength > 0.005 else "weak buy" if signal_strength > 0 else "strong sell" if signal_strength < -0.005 else "weak sell"
            signals[model_name] = category
            confidences[model_name] = abs(signal_strength) / latest_data['close'].iloc[-1]

        # LSTM Model
        lstm_features = scaled_features.reshape((scaled_features.shape[0], scaled_features.shape[1], 1))
        lstm_prediction = self.models['lstm'].predict(lstm_features)
        signal_strength_lstm = lstm_prediction[-1][0] - latest_data['close'].iloc[-1]
        category_lstm = "strong buy" if signal_strength_lstm > 0.005 else "weak buy" if signal_strength_lstm > 0 else "strong sell" if signal_strength_lstm < -0.005 else "weak sell"
        signals['lstm'] = category_lstm
        confidences['lstm'] = abs(signal_strength_lstm) / latest_data['close'].iloc[-1]

        # Hybrid CNN-LSTM Model
        hybrid_prediction = self.models['hybrid_cnn_lstm'].predict(lstm_features)
        signal_strength_hybrid = hybrid_prediction[-1][0] - latest_data['close'].iloc[-1]
        category_hybrid = "strong buy" if signal_strength_hybrid > 0.005 else "weak buy" if signal_strength_hybrid > 0 else "strong sell" if signal_strength_hybrid < -0.005 else "weak sell"
        signals['hybrid_cnn_lstm'] = category_hybrid
        confidences['hybrid_cnn_lstm'] = abs(signal_strength_hybrid) / latest_data['close'].iloc[-1]

        # ARIMA Model
        arima_forecast = self.models['arima'].forecast(steps=1)
        signal_strength_arima = arima_forecast[0] - latest_data['close'].iloc[-1]
        category_arima = "strong buy" if signal_strength_arima > 0.005 else "weak buy" if signal_strength_arima > 0 else "strong sell" if signal_strength_arima < -0.005 else "weak sell"
        signals['arima'] = category_arima
        confidences['arima'] = abs(signal_strength_arima) / latest_data['close'].iloc[-1]

        # Prophet Model
        prophet_forecast = self.models['prophet'].predict(pd.DataFrame({'ds': [latest_data.index[-1] + pd.Timedelta(minutes=5)]}))
        signal_strength_prophet = prophet_forecast['yhat'].iloc[0] - latest_data['close'].iloc[-1]
        category_prophet = "strong buy" if signal_strength_prophet > 0.005 else "weak buy" if signal_strength_prophet > 0 else "strong sell" if signal_strength_prophet < -0.005 else "weak sell"
        signals['prophet'] = category_prophet
        confidences['prophet'] = abs(signal_strength_prophet) / latest_data['close'].iloc[-1]

        return signals, confidences

    async def concurrent_signal_generation(self, symbol: str):
        tasks = []
        for timeframe in self.timeframes:
            data = await self.fetch_data(symbol, timeframe)
            preprocessed_data = self.preprocess_data(data)
            await self.train_models(preprocessed_data)
            task = self.generate_signals(preprocessed_data.iloc[-30:])
            tasks.append(task)

        results = await asyncio.gather(*tasks)
        return results

    async def save_signals_to_db(self, symbol: str, timeframe: str, final_decision: str, signals: Dict[str, str], confidences: Dict[str, float]):
        TradeSignal.objects.update_or_create(
            symbol=symbol,
            timeframe=timeframe,
            defaults={
                'decision': final_decision,
                'signals': json.dumps(signals),
                'confidences': json.dumps(confidences),
            }
        )
        logger.info(f"Signals saved for {symbol} {timeframe}")

    async def run(self):
        await self.initialize()
        for symbol in self.symbols:
            results = await self.concurrent_signal_generation(symbol)
            for timeframe, (signals, confidences) in zip(self.timeframes, results):
                final_decision = "buy" if sum(1 for s in signals.values() if 'buy' in s) > len(signals) / 2 else "sell"
                await self.save_signals_to_db(symbol, timeframe, final_decision, signals, confidences)
        await self.session.close()

if __name__ == "__main__":
    symbols = ['BTC/USDT', 'ETH/USDT']  # Example list of symbols
    bot = MachineTradingBot(symbols)
    asyncio.run(bot.run())

#mainscript.py
import asyncio
import logging

# Import required functions and classes from each module
from fetch_data import DataFetcher
from machine_model import MachineTradingBot
from order_book import OrderbookBot
from priceaction import PriceActionBot
from sentiment import SentimentAnalyzer
from technical_analysis import CryptoBot
from models import Signal

# Logger setup
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

async def main():
    # Define symbols and timeframes
    symbols = ['BTC/USDT', 'ETH/USDT']  # Example list of symbols
    timeframes = ['5m', '15m', '1h']

    # Initialize the analysis objects
    data_fetcher = DataFetcher(symbols=symbols, timeframes=timeframes)
    machine_bot = MachineTradingBot(symbols=symbols)
    orderbook_bot = OrderbookBot()
    price_action_bot = PriceActionBot(timeframes, lookback=10, db_config={})
    sentiment_analyzer = SentimentAnalyzer()
    technical_bot = CryptoBot(timeframes=timeframes)

    # Fetch the data
    await data_fetcher.run()  # This fetches historical and real-time data

    # Perform the analyses
    await machine_bot.run()
    await orderbook_bot.run_bot()
    await sentiment_analyzer.add_sentiment_features()
    await price_action_bot.run_bot()
    await technical_bot.run_bot()

    # Collect all signals
    all_signals = Signal.objects.all()

    combined_signals = {}
    for signal in all_signals:
        key = (signal.symbol, signal.timeframe)
        if key not in combined_signals:
            combined_signals[key] = []
        combined_signals[key].append(signal)

    # Compare and categorize signals
    categorized_signals = []
    for key, signal_list in combined_signals.items():
        symbol, timeframe = key
        signal_counts = {'Buy': 0, 'Sell': 0, 'Neutral': 0}
        analysis_sources = {'Buy': [], 'Sell': [], 'Neutral': []}

        for signal in signal_list:
            signal_counts[signal.signal_type] += 1
            analysis_sources[signal.signal_type].append(signal.analysis_type)  # Assuming signal includes analysis_type

        # Determine the strongest signal
        if signal_counts['Buy'] > signal_counts['Sell']:
            if signal_counts['Buy'] > 2:
                category = 'Strong Bullish'
            elif signal_counts['Buy'] > 1:
                category = 'Bullish'
            else:
                category = 'Weak Bullish'
        elif signal_counts['Sell'] > signal_counts['Buy']:
            if signal_counts['Sell'] > 2:
                category = 'Strong Bearish'
            elif signal_counts['Sell'] > 1:
                category = 'Bearish'
            else:
                category = 'Weak Bearish'
        else:
            category = 'Neutral'

        categorized_signal = {
            'symbol': symbol,
            'timeframe': timeframe,
            'category': category,
            'signal_sources': analysis_sources[category.split()[-1]] if category != 'Neutral' else []
        }
        categorized_signals.append(categorized_signal)

        logger.info(f"Processed signal for {symbol} {timeframe}: {category}")

    # Save categorized signals back to the database or a combined output
    for signal in categorized_signals:
        Signal.objects.create(
            symbol=signal['symbol'],
            timeframe=signal['timeframe'],
            signal_type=signal['category'],
            analysis_type=', '.join(signal['signal_sources'])
        )
        logger.info(f"Saved categorized signal for {signal['symbol']} {signal['timeframe']} as {signal['category']}")

if __name__ == "__main__":
    asyncio.run(main())

#orderbook.py
import asyncio
import logging
import os
import time
from typing import Dict, List, Tuple

import numpy as np
from scipy import stats
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from sklearn.preprocessing import StandardScaler
import pandas as pd

# Django setup
import django
from django.utils import timezone

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'binancebot.settings')
django.setup()

from .models import Kline, OrderBook, Signal, TradingPair  # Assuming these models exist

# Logger setup
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

class OrderbookBot:
    def __init__(self, short_period: int = 100, long_period: int = 500, volume_threshold: float = 2, ob_imbalance_threshold: float = 0.7):
        self.short_period = short_period
        self.long_period = long_period
        self.volume_threshold = volume_threshold
        self.ob_imbalance_threshold = ob_imbalance_threshold
        self.pair_data: Dict[str, Dict] = {}

    def initialize_pair(self, pair: str):
        self.pair_data[pair] = {
            'volume_history': [],
            'ob_imbalance_history': [],
            'price_history': [],
            'volatility_history': []
        }

    async def get_candles_df(self, pair: str, timeframe: str, num_candles: int = 200) -> pd.DataFrame:
        """Fetches recent candle data for the specified trading pair and timeframe."""
        klines = Kline.objects.filter(symbol=pair, timeframe=timeframe).order_by("-timestamp")[:num_candles]
        df = pd.DataFrame.from_records(klines.values())
        df.set_index("timestamp", inplace=True)
        df.sort_index(inplace=True)  # Ensure chronological order for analysis
        return df

    async def get_order_book(self, pair: str) -> Tuple[List[Tuple[float, float]], List[Tuple[float, float]]]:
        """Fetches the latest order book data for the specified trading pair."""
        try:
            order_book_record = OrderBook.objects.filter(symbol=pair).latest('timestamp')
            bids = order_book_record.bids
            asks = order_book_record.asks
            return bids, asks
        except OrderBook.DoesNotExist:
            logger.warning(f"No order book data for {pair}")
            return [], []

    def calculate_volume_zscore(self, pair: str, current_volume: float) -> float:
        self.pair_data[pair]['volume_history'].append(current_volume)
        if len(self.pair_data[pair]['volume_history']) < self.short_period:
            return 0
        return stats.zscore(self.pair_data[pair]['volume_history'])[-1]

    def calculate_ob_imbalance(self, bids: List[Tuple[float, float]], asks: List[Tuple[float, float]]) -> float:
        total_bid_volume = sum(bid[1] for bid in bids)
        total_ask_volume = sum(ask[1] for ask in asks)
        return total_bid_volume / (total_bid_volume + total_ask_volume)

    def calculate_ob_imbalance_zscore(self, pair: str, current_imbalance: float) -> float:
        self.pair_data[pair]['ob_imbalance_history'].append(current_imbalance)
        if len(self.pair_data[pair]['ob_imbalance_history']) < self.short_period:
            return 0
        return stats.zscore(self.pair_data[pair]['ob_imbalance_history'])[-1]

    def calculate_volatility(self, pair: str, current_price: float) -> float:
        self.pair_data[pair]['price_history'].append(current_price)
        if len(self.pair_data[pair]['price_history']) < self.short_period:
            return 0
        returns = np.diff(self.pair_data[pair]['price_history']) / self.pair_data[pair]['price_history'][:-1]
        return np.std(returns) * np.sqrt(len(returns))

    async def generate_signal(self, pair: str, timeframe: str) -> Tuple[str, Dict]:
        try:
            df = await self.get_candles_df(pair, timeframe)
            if df.empty:
                return "Neutral", {}

            current_volume = df['volume'].iloc[-1]
            current_price = df['close'].iloc[-1]

            # Get order book data
            bids, asks = await self.get_order_book(pair)

            if not bids or not asks:
                return "Neutral", {}

            ob_imbalance = self.calculate_ob_imbalance(bids, asks)
            volume_zscore = self.calculate_volume_zscore(pair, current_volume)
            ob_imbalance_zscore = self.calculate_ob_imbalance_zscore(pair, ob_imbalance)
            volatility = self.calculate_volatility(pair, current_price)

            # Determine adaptive thresholds
            market_regime = "medium_volatility"  # Simplified; enhance with models if needed
            adaptive_volume_threshold, adaptive_ob_threshold = self.adaptive_thresholds(market_regime)

            signal = "Neutral"
            if volume_zscore > adaptive_volume_threshold and ob_imbalance > adaptive_ob_threshold:
                if ob_imbalance_zscore > 0:
                    signal = "Buy"
                elif ob_imbalance_zscore < 0:
                    signal = "Sell"

            # Calculate support and resistance levels
            support_level, resistance_level = df['low'].min(), df['high'].max()
            stop_loss, take_profit = self.calculate_stop_loss_take_profit(signal, current_price, support_level, resistance_level)
            
            metrics = {
                'volume_zscore': volume_zscore,
                'ob_imbalance': ob_imbalance,
                'ob_imbalance_zscore': ob_imbalance_zscore,
                'volatility': volatility,
                'stop_loss': stop_loss,
                'take_profit': take_profit,
                'support': support_level,
                'resistance': resistance_level
            }

            logger.info(f"Generated signal for {pair}: {signal}")
            return signal, metrics
        except Exception as e:
            logger.error(f"Error generating signal for {pair}: {str(e)}")
            return "Neutral", {}

    def calculate_stop_loss_take_profit(self, signal: str, current_price: float, support: float, resistance: float) -> Tuple[float, float]:
        if signal == "Buy":
            return support, resistance
        elif signal == "Sell":
            return resistance, support
        else:
            return support, resistance

    async def save_signal(self, pair: str, timeframe: str, signal: str, metrics: Dict[str, float]):
        try:
            Signal.objects.create(
                symbol=pair,
                timestamp=timezone.now(),
                timeframe=timeframe,
                signal_type=signal,
                price=metrics.get("current_price"),
                stop_loss=metrics.get("stop_loss"),
                take_profit=metrics.get("take_profit")
            )
            logger.info(f"Signal for {pair} {timeframe} saved to the database.")
        except Exception as e:
            logger.error(f"Error saving signal for {pair} {timeframe}: {e}")

    async def run_bot(self):
        while True:
            logger.info("Starting bot cycle...")
            pairs = TradingPair.objects.values_list("symbol", flat=True)
            timeframes = ["1m", "5m", "15m", "1h"]
            tasks = []

            for pair in pairs:
                for timeframe in timeframes:
                    task = self.process_pair_timeframe(pair, timeframe)
                    tasks.append(task)

            await asyncio.gather(*tasks)

            time.sleep(60)  # Adjust sleep time according to your needs

    async def process_pair_timeframe(self, pair: str, timeframe: str):
        signal, metrics = await self.generate_signal(pair, timeframe)
        await self.save_signal(pair, timeframe, signal, metrics)

if __name__ == "__main__":
    bot = OrderbookBot()
    try:
        asyncio.run(bot.run_bot())
    except KeyboardInterrupt:
        logger.info("Stopping bot...")

#priceaction.py
import random
import pandas as pd
import numpy as np
from django.utils import timezone
from django.db.models import Max
from .models import Kline  # Adjust this import based on your Django app structure
import logging
import traceback
from concurrent.futures import ThreadPoolExecutor
from scipy.signal import argrelextrema
from collections import deque
import time
import asyncio
import psycopg2
from psycopg2 import sql
import telegram
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart

# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

class PriceActionBot:
    def __init__(self, intervals, lookback, db_config, telegram_token=None, email_config=None):
        self.intervals = intervals
        self.lookback = lookback
        self.db_config = db_config
        self.telegram_token = telegram_token
        self.email_config = email_config
        self.dfs = {}
        self.volatility = {}
        self.performance_metrics = {}

    async def get_real_time_data(self, symbol, interval):
        try:
            end_time = timezone.now()
            start_time = end_time - timezone.timedelta(days=self.lookback)
            
            klines = await Kline.objects.filter(
                symbol=symbol,
                interval=interval,
                open_time__gte=start_time,
                open_time__lte=end_time
            ).order_by('open_time').to_list()

            data = []
            for kline in klines:
                data.append([
                    kline.open_time,
                    kline.open_price,
                    kline.high_price,
                    kline.low_price,
                    kline.close_price,
                    kline.volume,
                    kline.close_time,
                    kline.quote_asset_volume,
                    kline.number_of_trades,
                    kline.taker_buy_base_asset_volume,
                    kline.taker_buy_quote_asset_volume
                ])

            df = pd.DataFrame(data, columns=[
                'timestamp', 'open', 'high', 'low', 'close', 'volume',
                'close_time', 'quote_asset_volume', 'number_of_trades',
                'taker_buy_base_asset_volume', 'taker_buy_quote_asset_volume'
            ])
            df['timestamp'] = pd.to_datetime(df['timestamp'])
            for col in ['open', 'high', 'low', 'close', 'volume']:
                df[col] = df[col].astype(float)
            df.set_index('timestamp', inplace=True)
            return df
        except Exception as e:
            logger.error(f"Error fetching real-time data for {symbol} on {interval}: {e}")
            return None

    async def calculate_adaptive_thresholds(self, df):
        volatility = df['close'].pct_change().rolling(window=20).std().iloc[-1]
        return {
            'weak': volatility * 0.5,
            'medium': volatility,
            'strong': volatility * 2
        }

    def detect_candlestick_patterns(self, df):
        patterns = {}
        
        # Existing patterns
        patterns['doji'] = abs(df['open'] - df['close']) <= (0.1 * (df['high'] - df['low']))
        patterns['hammer'] = ((df['high'] - df['low']) > 3 * (df['open'] - df['close'])) & \
                             ((df['close'] - df['low']) / (0.001 + df['high'] - df['low']) > 0.6) & \
                             ((df['open'] - df['low']) / (0.001 + df['high'] - df['low']) > 0.6)
        patterns['shooting_star'] = ((df['high'] - df['low']) > 3 * (df['open'] - df['close'])) & \
                                    ((df['high'] - df['close']) / (0.001 + df['high'] - df['low']) > 0.6) & \
                                    ((df['high'] - df['open']) / (0.001 + df['high'] - df['low']) > 0.6)
        patterns['bullish_engulfing'] = (df['open'].shift(1) > df['close'].shift(1)) & \
                                        (df['close'] > df['open'].shift(1)) & \
                                        (df['open'] < df['close'].shift(1)) & \
                                        (df['close'] > df['open'])
        patterns['bearish_engulfing'] = (df['close'].shift(1) > df['open'].shift(1)) & \
                                        (df['open'] > df['close'].shift(1)) & \
                                        (df['close'] < df['open'].shift(1)) & \
                                        (df['open'] > df['close'])
        patterns['bullish_harami'] = (df['open'].shift(1) > df['close'].shift(1)) & \
                                     (df['open'] > df['close']) & \
                                     (df['open'] <= df['close'].shift(1)) & \
                                     (df['close'] >= df['open'].shift(1))
        patterns['bearish_harami'] = (df['close'].shift(1) > df['open'].shift(1)) & \
                                     (df['close'] > df['open']) & \
                                     (df['open'] >= df['close'].shift(1)) & \
                                     (df['close'] <= df['open'].shift(1))
        
        # New patterns
        patterns['long_legged_doji'] = (abs(df['open'] - df['close']) <= (0.1 * (df['high'] - df['low']))) & \
                                       ((df['high'] - df['low']) > 3 * abs(df['open'] - df['close']))
        patterns['gravestone_doji'] = (abs(df['open'] - df['close']) <= (0.1 * (df['high'] - df['low']))) & \
                                      ((df['high'] - df['close']) > 3 * (df['close'] - df['low']))
        patterns['dragonfly_doji'] = (abs(df['open'] - df['close']) <= (0.1 * (df['high'] - df['low']))) & \
                                     ((df['close'] - df['low']) > 3 * (df['high'] - df['close']))
        patterns['hanging_man'] = ((df['high'] - df['low']) > 3 * (df['open'] - df['close'])) & \
                                  ((df['close'] - df['low']) / (0.001 + df['high'] - df['low']) > 0.6) & \
                                  ((df['open'] - df['low']) / (0.001 + df['high'] - df['low']) > 0.6) & \
                                  (df['close'].shift(1) > df['open'].shift(1))
        patterns['belt_hold'] = (abs(df['open'] - df['low']) <= (0.1 * (df['high'] - df['low']))) & \
                                (df['close'] > df['open'])
        patterns['marubozu'] = (abs(df['open'] - df['high']) <= (0.1 * (df['high'] - df['low']))) & \
                               (abs(df['close'] - df['low']) <= (0.1 * (df['high'] - df['low'])))
        patterns['three_white_soldiers'] = (df['close'] > df['open']) & \
                                           (df['close'].shift(1) > df['open'].shift(1)) & \
                                           (df['close'].shift(2) > df['open'].shift(2)) & \
                                           (df['open'] > df['close'].shift(1)) & \
                                           (df['open'].shift(1) > df['close'].shift(2))
        patterns['rising_three_methods'] = (df['close'].shift(4) > df['open'].shift(4)) & \
                                           (df['close'] > df['open']) & \
                                           (df['close'] > df['close'].shift(4)) & \
                                           (df['open'].shift(1) < df['close'].shift(4)) & \
                                           (df['close'].shift(1) < df['close'].shift(4)) & \
                                           (df['open'].shift(2) < df['close'].shift(4)) & \
                                           (df['close'].shift(2) < df['close'].shift(4)) & \
                                           (df['open'].shift(3) < df['close'].shift(4)) & \
                                           (df['close'].shift(3) < df['close'].shift(4))
        patterns['piercing_line'] = (df['close'].shift(1) < df['open'].shift(1)) & \
                                    (df['open'] < df['low'].shift(1)) & \
                                    (df['close'] > (df['open'].shift(1) + df['close'].shift(1)) / 2) & \
                                    (df['close'] < df['open'].shift(1))
        patterns['inverted_hammer'] = ((df['high'] - df['low']) > 3 * (df['open'] - df['close'])) & \
                                      ((df['high'] - df['open']) / (0.001 + df['high'] - df['low']) > 0.6) & \
                                      ((df['high'] - df['close']) / (0.001 + df['high'] - df['low']) > 0.6)
        
        return pd.DataFrame(patterns)

    def detect_head_and_shoulders(self, df, window=20):
        highs = df['high'].values
        lows = df['low'].values
        
        left_shoulder = argrelextrema(highs[:window], np.greater)[0]
        head = argrelextrema(highs[window:2*window], np.greater)[0] + window
        right_shoulder = argrelextrema(highs[2*window:], np.greater)[0] + 2*window
        
        if len(left_shoulder) > 0 and len(head) > 0 and len(right_shoulder) > 0:
            if highs[head] > highs[left_shoulder] and highs[head] > highs[right_shoulder]:
                neckline = min(lows[left_shoulder], lows[right_shoulder])
                if df['close'].iloc[-1] < neckline:
                    return 'Head and Shoulders Top'
        
        left_shoulder = argrelextrema(lows[:window], np.less)[0]
        head = argrelextrema(lows[window:2*window], np.less)[0] + window
        right_shoulder = argrelextrema(lows[2*window:], np.less)[0] + 2*window
        
        if len(left_shoulder) > 0 and len(head) > 0 and len(right_shoulder) > 0:
            if lows[head] < lows[left_shoulder] and lows[head] < lows[right_shoulder]:
                neckline = max(highs[left_shoulder], highs[right_shoulder])
                if df['close'].iloc[-1] > neckline:
                    return 'Head and Shoulders Bottom'
        
        return None

    def detect_triangle_pattern(self, df, window=20):
        highs = df['high'].rolling(window=5).max()
        lows = df['low'].rolling(window=5).min()
        
        upper_trend = np.polyfit(range(window), highs[-window:], 1)
        lower_trend = np.polyfit(range(window), lows[-window:], 1)
        
        if abs(upper_trend[0]) < 0.1 and abs(lower_trend[0]) < 0.1:
            return 'Symmetrical Triangle'
        elif upper_trend[0] < -0.1 and abs(lower_trend[0]) < 0.1:
            return 'Descending Triangle'
        elif abs(upper_trend[0]) < 0.1 and lower_trend[0] > 0.1:
            return 'Ascending Triangle'
        
        return None

    def detect_flags_pennants(self, df, window=20):
        trend = np.polyfit(range(window), df['close'][-window:], 1)[0]
        
        if abs(trend) > 0.1:
            highs = df['high'].rolling(window=5).max()
            lows = df['low'].rolling(window=5).min()
            
            upper_trend = np.polyfit(range(window), highs[-window:], 1)
            lower_trend = np.polyfit(range(window), lows[-window:], 1)
            
            if abs(upper_trend[0] - lower_trend[0]) < 0.05:
                return 'Flag'
            elif abs(upper_trend[0]) < abs(lower_trend[0]):
                return 'Pennant'
        
        return None

    def fibonacci_retracement(self, df, window=100):
        high = df['high'][-window:].max()
        low = df['low'][-window:].min()
        diff = high - low
        levels = [0, 0.236, 0.382, 0.5, 0.618, 0.786, 1]
        retracements = [high - l * diff for l in levels]
        return retracements

    def volume_analysis(self, df, window=20):
        volume = df['volume'].values
        price = df['close'].values
        
        avg_volume = np.mean(volume[-window:])
        volume_trend = 'Increasing' if volume[-1] > avg_volume else 'Decreasing'
        
        obv = np.cumsum(np.where(price > np.roll(price, 1), volume, -volume))
        obv_trend = 'Bullish' if obv[-1] > obv[-2] else 'Bearish'
        
        return volume_trend, obv_trend
    async def advanced_price_action_analysis(self, df, symbol, interval):
        try:
            recent_df = df.iloc[-200:].copy()
            recent_df.dropna(inplace=True)

            if len(recent_df) < 50:
                return None

            thresholds = await self.calculate_adaptive_thresholds(recent_df)


            candlestick_patterns = self.detect_candlestick_patterns(recent_df)
            recent_df = pd.concat([recent_df, candlestick_patterns], axis=1)

            head_and_shoulders = self.detect_head_and_shoulders(recent_df)
            triangle = self.detect_triangle_pattern(recent_df)
            flag_pennant = self.detect_flags_pennants(recent_df)

            short_term_trend = np.polyfit(range(20), recent_df['close'].values[-20:], 1)[0]
            medium_term_trend = np.polyfit(range(50), recent_df['close'].values[-50:], 1)[0]
            long_term_trend = np.polyfit(range(100), recent_df['close'].values[-100:], 1)[0]

            support = np.min(recent_df['low'].values[-20:])
            resistance = np.max(recent_df['high'].values[-20:])

            fib_levels = self.fibonacci_retracement(recent_df)

            volume_trend, obv_trend = self.volume_analysis(recent_df)

            signal = 'NO_SIGNAL'
            strength = 0

            # Trend-based signals
            if short_term_trend > 0 and medium_term_trend > 0 and long_term_trend > 0:
                signal = 'BUY'
                strength += 1
            elif short_term_trend < 0 and medium_term_trend < 0 and long_term_trend < 0:
                signal = 'SELL'
                strength += 1

            # Candlestick pattern signals
            bullish_patterns = ['bullish_engulfing', 'hammer', 'bullish_harami', 'three_white_soldiers', 'piercing_line', 'inverted_hammer']
            bearish_patterns = ['bearish_engulfing', 'shooting_star', 'bearish_harami', 'hanging_man', 'gravestone_doji']
            
            for pattern in bullish_patterns:
                if recent_df[pattern].iloc[-1]:
                    if signal == 'BUY':
                        strength += 1
                    else:
                        signal = 'BUY'
                        strength = 1
                    break
            
            for pattern in bearish_patterns:
                if recent_df[pattern].iloc[-1]:
                    if signal == 'SELL':
                        strength += 1
                    else:
                        signal = 'SELL'
                        strength = 1
                    break

            # Chart pattern signals
            if head_and_shoulders == 'Head and Shoulders Top':
                if signal == 'SELL':
                    strength += 2
                else:
                    signal = 'SELL'
                    strength = 2
            elif head_and_shoulders == 'Head and Shoulders Bottom':
                if signal == 'BUY':
                    strength += 2
                else:
                    signal = 'BUY'
                    strength = 2

            if triangle == 'Ascending Triangle' and signal == 'BUY':
                strength += 1
            elif triangle == 'Descending Triangle' and signal == 'SELL':
                strength += 1

            if flag_pennant in ['Flag', 'Pennant']:
                if signal == 'BUY' and short_term_trend > 0:
                    strength += 1
                elif signal == 'SELL' and short_term_trend < 0:
                    strength += 1

            # Volume confirmation
            if volume_trend == 'Increasing' and obv_trend == 'Bullish' and signal == 'BUY':
                strength += 1
            elif volume_trend == 'Increasing' and obv_trend == 'Bearish' and signal == 'SELL':
                strength += 1

            # Categorize signal strength

            if strength >= thresholds['strong']:
                signal = f'STRONG_{signal}'
            elif strength >= thresholds['medium']:
                signal = f'{signal}'
            elif strength >= thresholds['weak']:
                signal = f'WEAK_{signal}'
            else:
                signal = 'NO_SIGNAL'

            last_row = recent_df.iloc[-1].copy()
            last_row['Signal'] = signal
            last_row['Signal_Strength'] = strength
            last_row['Symbol'] = symbol
            last_row['Interval'] = interval

            return last_row.to_frame().T

        except Exception as e:
            logger.error(f"Error in advanced price action analysis: {e}\n{traceback.format_exc()}")
            return None

    async def save_signal_to_db(self, signal_data):
        try:
            conn = psycopg2.connect(**self.db_config)
            cur = conn.cursor()
            
            columns = signal_data.columns.tolist()
            values = [signal_data[col].iloc[0] for col in columns]
            
            insert_query = sql.SQL("INSERT INTO signals ({}) VALUES ({})").format(
                sql.SQL(', ').join(map(sql.Identifier, columns)),
                sql.SQL(', ').join(sql.Placeholder() * len(columns))
            )
            
            cur.execute(insert_query, values)
            conn.commit()
            
            cur.close()
            conn.close()
            
            logger.info(f"Signal saved to database: {signal_data['Signal'].iloc[0]}")
        except Exception as e:
            logger.error(f"Error saving signal to database: {e}\n{traceback.format_exc()}")

    async def send_telegram_alert(self, message):
        if self.telegram_token:
            try:
                bot = telegram.Bot(token=self.telegram_token)
                await bot.send_message(chat_id=self.telegram_chat_id, text=message)
                logger.info("Telegram alert sent successfully")
            except Exception as e:
                logger.error(f"Error sending Telegram alert: {e}\n{traceback.format_exc()}")

    async def send_email_alert(self, subject, body):
        if self.email_config:
            try:
                msg = MIMEMultipart()
                msg['From'] = self.email_config['sender']
                msg['To'] = self.email_config['recipient']
                msg['Subject'] = subject
                
                msg.attach(MIMEText(body, 'plain'))
                
                server = smtplib.SMTP(self.email_config['smtp_server'], self.email_config['smtp_port'])
                server.starttls()
                server.login(self.email_config['username'], self.email_config['password'])
                server.send_message(msg)
                server.quit()
                
                logger.info("Email alert sent successfully")
            except Exception as e:
                logger.error(f"Error sending email alert: {e}\n{traceback.format_exc()}")

    async def analyze_interval(self, interval):
        while True:
            try:
                df = await self.get_real_time_data(interval)
                if df is not None:
                    result = await self.advanced_price_action_analysis(df, interval)
                    if result is not None:
                        await self.save_signal_to_db(result)
                        
                        signal = result['Signal'].iloc[0]
                        strength = result['Signal_Strength'].iloc[0]
                        
                        if signal != 'NO_SIGNAL':
                            message = f"New signal for {self.symbol} on {interval} timeframe: {signal} (Strength: {strength})"
                            await self.send_telegram_alert(message)
                            await self.send_email_alert("New Trading Signal", message)
                    
                        logger.info(f"Analysis Result for {interval}:\n{result}")
                    else:
                        logger.warning(f"No analysis result generated for {interval}")
                else:
                    logger.warning(f"No historical data available for {interval}")
                
                await asyncio.sleep(self.get_interval_seconds(interval))
                
            except Exception as e:
                logger.error(f"Error in analysis loop for {interval}: {e}\n{traceback.format_exc()}")
                await asyncio.sleep(60)

    def get_interval_seconds(self, interval):
        interval_map = {
            '1m': 60,
            '5m': 300,
            '15m': 900,
            '30m': 1800,
            '1h': 3600,
            '4h': 14400,
            '1d': 86400
        }
        return interval_map.get(interval, 300)

    async def analyze_pair(self, symbol, interval):
        while True:
            try:
                df = await self.get_real_time_data(symbol, interval)
                if df is not None:
                    result = await self.advanced_price_action_analysis(df, symbol, interval)
                    if result is not None:
                        await self.save_signal_to_db(result)
                        
                        signal = result['Signal'].iloc[0]
                        strength = result['Signal_Strength'].iloc[0]
                        
                        if signal != 'NO_SIGNAL':
                            message = f"New signal for {symbol} on {interval} timeframe: {signal} (Strength: {strength})"
                            await self.send_telegram_alert(message)
                            await self.send_email_alert("New Trading Signal", message)
                    
                        logger.info(f"Analysis Result for {symbol} on {interval}:\n{result}")
                        
                        # Update performance metrics
                        await self.update_performance_metrics(symbol, interval, result)
                    else:
                        logger.warning(f"No analysis result generated for {symbol} on {interval}")
                else:
                    logger.warning(f"No real-time data available for {symbol} on {interval}")
                
                await asyncio.sleep(self.get_interval_seconds(interval))
                
            except Exception as e:
                logger.error(f"Error in analysis loop for {symbol} on {interval}: {e}\n{traceback.format_exc()}")
                await asyncio.sleep(60)

    async def update_performance_metrics(self, symbol, interval, result):
        if symbol not in self.performance_metrics:
            self.performance_metrics[symbol] = {}
        
        if interval not in self.performance_metrics[symbol]:
            self.performance_metrics[symbol][interval] = {
                'total_signals': 0,
                'correct_signals': 0,
                'incorrect_signals': 0,
                'profit_loss': 0,
            }
        
        metrics = self.performance_metrics[symbol][interval]
        metrics['total_signals'] += 1
        
        # Simulated performance tracking (replace with actual logic based on your requirements)
        if result['Signal'].iloc[0] != 'NO_SIGNAL':
            # Simulate a trade outcome (replace with actual trade tracking logic)
            trade_outcome = random.choice(['win', 'loss'])
            if trade_outcome == 'win':
                metrics['correct_signals'] += 1
                metrics['profit_loss'] += random.uniform(0.5, 2.0)
            else:
                metrics['incorrect_signals'] += 1
                metrics['profit_loss'] -= random.uniform(0.5, 1.0)
        
        logger.info(f"Updated performance metrics for {symbol} on {interval}: {metrics}")

    async def run_bot(self):
        # Fetch all USDT trading pairs from the database
        usdt_pairs = await self.get_usdt_pairs()
        
        tasks = []
        for symbol in usdt_pairs:
            for interval in self.intervals:
                tasks.append(self.analyze_pair(symbol, interval))
        
        await asyncio.gather(*tasks)

    async def get_usdt_pairs(self):
        try:
            # Fetch all unique symbols ending with USDT from the Kline model
            usdt_pairs = await Kline.objects.filter(symbol__endswith='USDT').values_list('symbol', flat=True).distinct()
            return list(usdt_pairs)
        except Exception as e:
            logger.error(f"Error fetching USDT pairs: {e}\n{traceback.format_exc()}")
            return []

if __name__ == "__main__":
    intervals = ["1m", "5m", "15m", "1h", "4h", "1d"]
    lookback = 30  # days

    db_config = {
        "host": "localhost",
        "database": "trading_bot",
        "user": "your_username",
        "password": "your_password"
    }

    telegram_token = "your_telegram_bot_token"
    telegram_chat_id = "your_telegram_chat_id"

    email_config = {
        "sender": "your_email@example.com",
        "recipient": "recipient@example.com",
        "smtp_server": "smtp.example.com",
        "smtp_port": 587,
        "username": "your_email@example.com",
        "password": "your_email_password"
    }

    bot = PriceActionBot(intervals, lookback, db_config, telegram_token, email_config)
    asyncio.run(bot.run_bot())

#sentiment.py
import requests
from bs4 import BeautifulSoup
from textblob import TextBlob
import pandas as pd
import time
from newspaper import Article
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import concurrent.futures
import logging
import json
import os
from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, func, Text
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
import numpy as np
from datetime import datetime, timedelta
from urllib.parse import urlparse
import re
from tenacity import retry, stop_after_attempt, wait_exponential
from statsmodels.tsa.seasonal import seasonal_decompose
from functools import lru_cache
from ratelimit import limits, sleep_and_retry

# Additional imports for new features
import tweepy  # For Twitter API
import praw    # For Reddit API
from telethon import TelegramClient
import nltk
from nltk.corpus import stopwords  # For NLP preprocessing
from nltk.tokenize import word_tokenize
import gensim
from gensim.summarization.summarizer import summarize  # For NLG
import spacy  # For NLU
import webbrowser

# Ensure NLTK resources are downloaded
nltk.download('stopwords')
nltk.download('punkt')

# Load spacy model
nlp = spacy.load('en_core_web_sm')

Base = declarative_base()

class SentimentData(Base):
    __tablename__ = 'sentiment_data'
    id = Column(Integer, primary_key=True)
    url = Column(String, unique=True)
    sentiment = Column(Float)
    timestamp = Column(DateTime)
    source = Column(String)  # Added source field to indicate the data source

class CacheData(Base):
    __tablename__ = 'cache_data'
    url = Column(String, primary_key=True)
    content = Column(Text)
    timestamp = Column(DateTime)

class TradingSignal(Base):
    __tablename__ = 'trading_signals'
    id = Column(Integer, primary_key=True)
    signal = Column(String)
    confidence = Column(Float)
    timestamp = Column(DateTime)

class SentimentAnalyzer:
    def __init__(self, config_path='config.json'):
        self.load_config(config_path)
        self.vader_analyzer = SentimentIntensityAnalyzer()
        self.session = requests.Session()
        self.session.headers.update({'User-Agent': 'Mozilla/5.0'})
        logging.basicConfig(level=logging.INFO, filename='sentiment_analysis.log')
        self.db_engine = create_engine(self.config['database_url'])
        Base.metadata.create_all(self.db_engine)
        self.db_session = sessionmaker(bind=self.db_engine)()
        self.domain_last_request = {}

        # Initialize Twitter API
        self.twitter_api = self.init_twitter_api()
        # Initialize Reddit API
        self.reddit_api = self.init_reddit_api()
        # Initialize Telegram Client
        self.telegram_client = self.init_telegram_client()
        # Initialize stopwords
        self.stop_words = set(stopwords.words('english'))

    def load_config(self, config_path):
        try:
            with open(config_path, 'r') as f:
                self.config = json.load(f)
        except FileNotFoundError:
            logging.error(f"Config file not found: {config_path}")
            raise
        except json.JSONDecodeError:
            logging.error(f"Invalid JSON in config file: {config_path}")
            raise

    # Initialize Twitter API
    def init_twitter_api(self):
        try:
            auth = tweepy.OAuth1UserHandler(
                self.config['twitter_api_key'],
                self.config['twitter_api_secret'],
                self.config['twitter_access_token'],
                self.config['twitter_access_token_secret']
            )
            api = tweepy.API(auth)
            return api
        except Exception as e:
            logging.error(f"Error initializing Twitter API: {str(e)}")
            return None

    # Initialize Reddit API
    def init_reddit_api(self):
        try:
            reddit = praw.Reddit(
                client_id=self.config['reddit_client_id'],
                client_secret=self.config['reddit_client_secret'],
                user_agent=self.config['reddit_user_agent']
            )
            return reddit
        except Exception as e:
            logging.error(f"Error initializing Reddit API: {str(e)}")
            return None

    # Initialize Telegram Client
    def init_telegram_client(self):
        try:
            client = TelegramClient(
                self.config['telegram_session_name'],
                self.config['telegram_api_id'],
                self.config['telegram_api_hash']
            )
            client.start()
            return client
        except Exception as e:
            logging.error(f"Error initializing Telegram Client: {str(e)}")
            return None

    @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
    @sleep_and_retry
    @limits(calls=1, period=1)  # Limit to 1 call per second
    def fetch_news_api(self, query="cryptocurrency", page_size=5):
        url = f"https://newsapi.org/v2/everything?q={query}&apiKey={self.config['news_api_key']}&pageSize={page_size}"
        try:
            response = self.session.get(url, timeout=10)
            response.raise_for_status()
            return response.json()['articles']
        except requests.RequestException as e:
            logging.error(f"News API request failed: {str(e)}")
            raise

    # Fetch articles from additional news sources
    def fetch_additional_news(self, query="cryptocurrency"):
        articles = []
        for source in self.config.get('additional_news_sources', []):
            try:
                response = self.session.get(source.format(query=query), timeout=10)
                response.raise_for_status()
                soup = BeautifulSoup(response.text, 'html.parser')
                links = [a['href'] for a in soup.find_all('a', href=True) if query.lower() in a.text.lower()]
                articles.extend(links)
            except Exception as e:
                logging.error(f"Error fetching from source {source}: {str(e)}")
        return articles

    @lru_cache(maxsize=100)
    def scrape_article(self, url):
        domain = urlparse(url).netloc
        self.rate_limit(domain)
        
        # Check cache first
        cached_content = self.get_from_cache(url)
        if cached_content:
            return cached_content

        try:
            article = Article(url)
            article.download()
            article.parse()
            content = article.text
            self.add_to_cache(url, content)
            return content
        except Exception as e:
            logging.error(f"Error scraping {url}: {str(e)}")
            return ""

    def get_from_cache(self, url):
        cache_data = self.db_session.query(CacheData).filter(
            CacheData.url == url,
            CacheData.timestamp > datetime.now() - timedelta(days=1)
        ).first()
        return cache_data.content if cache_data else None

    def add_to_cache(self, url, content):
        cache_data = CacheData(url=url, content=content, timestamp=datetime.now())
        self.db_session.merge(cache_data)
        self.db_session.commit()

    def clear_cache(self):
        self.scrape_article.cache_clear()  # Clear the LRU cache
        # Clear old entries from the database cache
        self.db_session.query(CacheData).filter(
            CacheData.timestamp <= datetime.now() - timedelta(days=7)
        ).delete()
        self.db_session.commit()

    def rate_limit(self, domain):
        current_time = time.time()
        if domain in self.domain_last_request:
            elapsed = current_time - self.domain_last_request[domain]
            if elapsed < self.config['per_domain_rate_limit']:
                time.sleep(self.config['per_domain_rate_limit'] - elapsed)
        self.domain_last_request[domain] = time.time()

    def preprocess_text(self, text):
        # Additional preprocessing steps
        text = re.sub(r'http\S+', '', text)  # Remove URLs
        text = re.sub(r'[^a-zA-Z\s]', '', text)  # Remove non-alphabetic characters
        text = text.lower()
        # Remove stopwords
        words = word_tokenize(text)
        filtered_words = [word for word in words if word not in self.stop_words]
        return ' '.join(filtered_words)

    def get_sentiment(self, text):
        try:
            preprocessed_text = self.preprocess_text(text)
            blob_sentiment = TextBlob(preprocessed_text).sentiment.polarity
            vader_sentiment = self.vader_analyzer.polarity_scores(preprocessed_text)['compound']
            return self.config['textblob_weight'] * blob_sentiment + self.config['vader_weight'] * vader_sentiment
        except Exception as e:
            logging.error(f"Error in sentiment analysis: {str(e)}")
            return 0

    def analyze_text_nlu(self, text):
        doc = nlp(text)
        entities = [(ent.text, ent.label_) for ent in doc.ents]
        return entities

    def process_url(self, url):
        text = self.scrape_article(url)
        sentiment = self.get_sentiment(text)
        insights = self.analyze_text_nlu(text)
        return {
            'url': url, 
            'sentiment': sentiment, 
            'timestamp': datetime.now(), 
            'source': 'News',
            'insights': insights
        }

    # New method to fetch and process Twitter data
    def fetch_twitter_data(self, query, count=100):
        tweets = []
        if self.twitter_api:
            try:
                for tweet in tweepy.Cursor(self.twitter_api.search_tweets, q=query, lang='en').items(count):
                    tweets.append(tweet.text)
                return tweets
            except Exception as e:
                logging.error(f"Error fetching tweets: {str(e)}")
        return tweets

    # New method to fetch and process Reddit data
    def fetch_reddit_data(self, subreddit_name='CryptoCurrency', limit=100):
        posts = []
        if self.reddit_api:
            try:
                subreddit = self.reddit_api.subreddit(subreddit_name)
                for post in subreddit.hot(limit=limit):
                    posts.append(post.title + ' ' + post.selftext)
            except Exception as e:
                logging.error(f"Error fetching Reddit posts: {str(e)}")
        return posts

    # New method to fetch and process Telegram data (public channels and groups)
    def fetch_telegram_data(self, query='cryptocurrency', limit=100):
        messages = []
        if self.telegram_client:
            try:
                async def get_messages():
                    async for dialog in self.telegram_client.iter_dialogs():
                        if dialog.is_channel or dialog.is_group:
                            async for message in self.telegram_client.iter_messages(dialog, search=query, limit=limit):
                                if message.message:
                                    messages.append(message.message)
                self.telegram_client.loop.run_until_complete(get_messages())
            except Exception as e:
                logging.error(f"Error fetching Telegram messages: {str(e)}")
        return messages

    # New method to perform data quality analysis
    def analyze_data_quality(self, data):
        quality_report = {}
        total_records = len(data)
        if total_records == 0:
            return {"error": "No data to analyze"}
        null_values = data.isnull().sum().sum()
        duplicate_records = data.duplicated().sum()
        completeness = ((total_records - null_values) / total_records) * 100
        uniqueness = ((total_records - duplicate_records) / total_records) * 100
        quality_report['total_records'] = total_records
        quality_report['null_values'] = null_values
        quality_report['duplicate_records'] = duplicate_records
        quality_report['completeness'] = completeness
        quality_report['uniqueness'] = uniqueness
        return quality_report

    # NLG method to generate detailed reports
    def generate_report(self, trend_report):
        try:
            # Summarize the trend report using NLG
            summary = summarize(trend_report, ratio=0.5)
            return summary
        except Exception as e:
            logging.error(f"Error generating report: {str(e)}")
            return trend_report

    def analyze_social_media_sentiment(self, query='cryptocurrency'):
        # Fetch data from Twitter
        twitter_data = self.fetch_twitter_data(query, count=100)
        twitter_sentiments = []
        for text in twitter_data:
            sentiment = self.get_sentiment(text)
            insights = self.analyze_text_nlu(text)
            twitter_sentiments.append({'text': text, 'sentiment': sentiment, 'timestamp': datetime.now(), 'source': 'Twitter', 'insights': insights})

        # Fetch data from Reddit
        reddit_data = self.fetch_reddit_data(limit=100)
        reddit_sentiments = []
        for text in reddit_data:
            sentiment = self.get_sentiment(text)
            insights = self.analyze_text_nlu(text)
            reddit_sentiments.append({'text': text, 'sentiment': sentiment, 'timestamp': datetime.now(), 'source': 'Reddit', 'insights': insights})

        # Fetch data from Telegram (public channels and groups)
        telegram_data = self.fetch_telegram_data(query=query, limit=100)
        telegram_sentiments = []
        for text in telegram_data:
            sentiment = self.get_sentiment(text)
            insights = self.analyze_text_nlu(text)
            telegram_sentiments.append({'text': text, 'sentiment': sentiment, 'timestamp': datetime.now(), 'source': 'Telegram', 'insights': insights})

        # Combine all sentiments
        all_social_sentiments = twitter_sentiments + reddit_sentiments + telegram_sentiments
        return pd.DataFrame(all_social_sentiments)

    def add_sentiment_features(self, data=None, query="cryptocurrency", page_size=5):
        if not isinstance(query, str) or not isinstance(page_size, int) or page_size <= 0:
            raise ValueError("Invalid input parameters")

        if data is None:
            data = pd.DataFrame()

        # Fetch news articles
        try:
            api_articles = self.fetch_news_api(query, page_size)
            api_urls = [article['url'] for article in api_articles]
            # Append additional news sources
            additional_articles = self.fetch_additional_news(query)
            api_urls.extend(additional_articles)
        except Exception as e:
            logging.error(f"Failed to fetch news articles: {str(e)}")
            api_urls = []  # Fallback to empty list if API fails

        all_urls = list(set(api_urls + self.config['crypto_sites']))

        results = []
        with concurrent.futures.ThreadPoolExecutor(max_workers=self.config['max_workers']) as executor:
            future_to_url = {executor.submit(self.process_url, url): url for url in all_urls}
            for future in concurrent.futures.as_completed(future_to_url):
                url = future_to_url[future]
                try:
                    result = future.result()
                    results.append(result)
                except Exception as e:
                    logging.error(f"Error processing {url}: {str(e)}")

        sentiment_df = pd.DataFrame(results)

        # Analyze social media sentiment
        social_sentiment_df = self.analyze_social_media_sentiment(query)

        # Combine news and social media sentiment
        combined_sentiment_df = pd.concat([sentiment_df, social_sentiment_df], ignore_index=True)

        # Perform data quality analysis
        data_quality_report = self.analyze_data_quality(combined_sentiment_df)
        logging.info(f"Data Quality Report: {data_quality_report}")

        if not data.empty:
            result = pd.concat([data, combined_sentiment_df], axis=1)
        else:
            result = combined_sentiment_df

        self.save_to_database(result)
        return result

    def save_to_database(self, data):
        try:
            data_records = data.to_dict(orient='records')
            # Optimize database interaction with batch inserts
            sentiment_data_objects = [SentimentData(
                url=record.get('url', ''),
                sentiment=record['sentiment'],
                timestamp=record['timestamp'],
                source=record.get('source', 'Unknown')
            ) for record in data_records]
            self.db_session.bulk_save_objects(sentiment_data_objects)
            self.db_session.commit()
        except Exception as e:
            self.db_session.rollback()
            logging.error(f"Error saving to database: {str(e)}")

    def analyze_trends(self, data, window_size='7D'):
        avg_sentiment = data['sentiment'].mean()
        std_sentiment = data['sentiment'].std()
        sentiment_trend = "Positive" if avg_sentiment > 0 else "Negative" if avg_sentiment < 0 else "Neutral"

        # Identify outliers
        outliers = data[np.abs(data['sentiment'] - avg_sentiment) > 2 * std_sentiment]

        # Time series analysis
        data['timestamp'] = pd.to_datetime(data['timestamp'])
        data = data.set_index('timestamp')
        daily_sentiment = data.resample(window_size)['sentiment'].mean()

        # Calculate moving average
        ma = daily_sentiment.rolling(window=self.config['moving_average_window']).mean()

        # Perform seasonal decomposition
        try:
            decomposition = seasonal_decompose(daily_sentiment.dropna(), model='additive', period=7)
            trend = decomposition.trend
            seasonal = decomposition.seasonal
            residual = decomposition.resid
        except Exception as e:
            logging.error(f"Error in seasonal decomposition: {str(e)}")
            trend = seasonal = residual = pd.Series()

        # Generate trading signals
        trading_signals = self.generate_trading_signals(data)

        # Save trading signals to database
        self.save_trading_signals(trading_signals)

        trend_report = f"""
        Overall sentiment trend: {sentiment_trend}
        Average sentiment: {avg_sentiment:.2f}
        Standard deviation: {std_sentiment:.2f}
        Number of outliers: {len(outliers)}
        Data Quality Report: {self.analyze_data_quality(data)}
        Daily sentiment trend:
        {daily_sentiment.to_string()}
        Moving Average ({self.config['moving_average_window']} periods):
        {ma.to_string()}
        Trend component:
        {trend.to_string()}
        Seasonal component:
        {seasonal.to_string()}
        Residual component:
        {residual.to_string()}
        """

        # Generate detailed report using NLG
        detailed_report = self.generate_report(trend_report)

        # Generate HTML report
        self.generate_html_report(trading_signals, detailed_report)

        return detailed_report

    def generate_trading_signals(self, data):
        trading_signals = []
        recent_sentiment = data['sentiment'].tail(self.config.get('signal_window', 10))
        avg_recent_sentiment = recent_sentiment.mean()

        if avg_recent_sentiment > self.config.get('bullish_threshold', 0.1):
            signal = 'Bullish'
            confidence = avg_recent_sentiment
        elif avg_recent_sentiment < self.config.get('bearish_threshold', -0.1):
            signal = 'Bearish'
            confidence = -avg_recent_sentiment
        else:
            signal = 'Neutral'
            confidence = avg_recent_sentiment

        trading_signal = {
            'signal': signal,
            'confidence': confidence,
            'timestamp': datetime.now()
        }
        trading_signals.append(trading_signal)

        return trading_signals

    def save_trading_signals(self, signals):
        try:
            signal_objects = [TradingSignal(
                signal=signal['signal'],
                confidence=signal['confidence'],
                timestamp=signal['timestamp']
            ) for signal in signals]
            self.db_session.bulk_save_objects(signal_objects)
            self.db_session.commit()
        except Exception as e:
            self.db_session.rollback()
            logging.error(f"Error saving trading signals to database: {str(e)}")

    def generate_html_report(self, signals, detailed_report):
        # Load HTML template
        with open('sentiment_results.html', 'r', encoding='utf-8') as f:
            html_template = f.read()

        # Generate signal tables
        bullish_table = self.generate_signal_table('Bullish')
        bearish_table = self.generate_signal_table('Bearish')
        neutral_table = self.generate_signal_table('Neutral')

        # Replace placeholders in the template
        html_content = html_template.format(
            detailed_report=detailed_report,
            bullish_signals=bullish_table,
            bearish_signals=bearish_table,
            neutral_signals=neutral_table
        )

        # Save the populated HTML
        with open('sentiment_analysis_report.html', 'w', encoding='utf-8') as f:
            f.write(html_content)

        # Optionally open the report in a web browser
        # webbrowser.open('sentiment_analysis_report.html')

    def generate_signal_table(self, signal_type):
        # Fetch signals from the database
        signals = self.db_session.query(TradingSignal).filter(TradingSignal.signal == signal_type).all()
        if not signals:
            return f"<p>No {signal_type} signals generated.</p>"
        table_html = "<table border='1' cellpadding='5' cellspacing='0'><tr><th>Signal</th><th>Confidence</th><th>Timestamp</th></tr>"
        for s in signals:
            table_html += f"<tr><td>{s.signal}</td><td>{s.confidence:.2f}</td><td>{s.timestamp.strftime('%Y-%m-%d %H:%M:%S')}</td></tr>"
        table_html += "</table>"
        return table_html

# Usage
if __name__ == "__main__":
    analyzer = SentimentAnalyzer()
    analyzer.clear_cache()  # Clear cache before starting
    result_data = analyzer.add_sentiment_features(query="bitcoin", page_size=10)
    trend_analysis = analyzer.analyze_trends(result_data)
    print(trend_analysis)

#technical_analysis.py
import asyncio
import logging
import time
from typing import Dict, List, Tuple

import pandas as pd
from ta.momentum import RSIIndicator, StochasticOscillator, AroonOscillator
from ta.trend import ADXIndicator, CCIIndicator, MACD, SMAIndicator, IchimokuCloud
from ta.volatility import BollingerBands

# Django setup
import os
import django

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'binancebot.settings')
django.setup()

from .models import Kline, Signal, TradingPair

# Logger setup
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

class CryptoBot:
    def __init__(self, timeframes: List[str] = ["5m", "15m", "1h"], num_candles: int = 200):
        self.timeframes = timeframes
        self.num_candles = num_candles
        logger.info("CryptoBot initialized with timeframes: %s and num_candles: %d", timeframes, num_candles)

    async def get_candles_df(self, pair: str, timeframe: str) -> pd.DataFrame:
        """
        Fetches the most recent `num_candles` records for the specified trading pair and timeframe.
        """
        klines = Kline.objects.filter(symbol=pair, timeframe=timeframe).order_by("-timestamp")[:self.num_candles]
        df = pd.DataFrame.from_records(klines.values())
        df.set_index("timestamp", inplace=True)
        df.sort_index(inplace=True)  # Ensure chronological order for analysis
        return df

    async def calculate_technical_indicators(self, df: pd.DataFrame) -> pd.DataFrame:
        # Technical indicators calculation
        df["sma_fast"] = SMAIndicator(close=df["close"], window=14).sma_indicator()
        df["sma_slow"] = SMAIndicator(close=df["close"], window=28).sma_indicator()
        df["rsi"] = RSIIndicator(close=df["close"], window=14).rsi()
        stoch = StochasticOscillator(close=df["close"], high=df["high"], low=df["low"], window=14, smooth_window=3)
        df["stoch_k"] = stoch.stoch()
        df["stoch_d"] = stoch.stoch_signal()

        macd = MACD(close=df["close"], window_slow=26, window_fast=12, window_sign=9)
        df["macd"] = macd.macd()
        df["macd_signal"] = macd.macd_signal()
        df["adx"] = ADXIndicator(high=df["high"], low=df["low"], close=df["close"], window=14).adx()
        df["cci"] = CCIIndicator(high=df["high"], low=df["low"], close=df["close"], window=20).cci()

        bollinger = BollingerBands(close=df["close"], window=20, window_dev=2)
        df["bollinger_mavg"] = bollinger.bollinger_mavg()
        df["bollinger_hband"] = bollinger.bollinger_hband()
        df["bollinger_lband"] = bollinger.bollinger_lband()

        ichimoku = IchimokuCloud(high=df["high"], low=df["low"], window1=9, window2=26, window3=52)
        df["ichimoku_a"] = ichimoku.ichimoku_a()
        df["ichimoku_b"] = ichimoku.ichimoku_b()

        aroon = AroonOscillator(high=df["high"], low=df["low"], window=14)
        df["aroon_up"] = aroon.aroon_up()
        df["aroon_down"] = aroon.aroon_down()

        return df

    async def generate_signal(self, pair: str, timeframe: str) -> Tuple[str, Dict[str, float]]:
        try:
            df = await self.get_candles_df(pair, timeframe)
            df = await self.calculate_technical_indicators(df)

            current_price = df["close"].iloc[-1]
            signal = "Neutral"

            # Example trading strategy (including multiple checks)
            if (
                df["sma_fast"].iloc[-2] < df["sma_slow"].iloc[-2] and df["sma_fast"].iloc[-1] > df["sma_slow"].iloc[-1]
                and df["rsi"].iloc[-1] < 30
                and df["stoch_k"].iloc[-1] < 20
                and df["stoch_d"].iloc[-1] < 20
            ):
                signal = "Strong Buy"
            elif df["sma_fast"].iloc[-2] < df["sma_slow"].iloc[-2] and df["sma_fast"].iloc[-1] > df["sma_slow"].iloc[-1]:
                signal = "Buy"
            elif (
                df["sma_fast"].iloc[-2] > df["sma_slow"].iloc[-2] and df["sma_fast"].iloc[-1] < df["sma_slow"].iloc[-1]
                and df["rsi"].iloc[-1] > 70
                and df["stoch_k"].iloc[-1] > 80
                and df["stoch_d"].iloc[-1] > 80
            ):
                signal = "Strong Sell"
            elif df["sma_fast"].iloc[-2] > df["sma_slow"].iloc[-2] and df["sma_fast"].iloc[-1] < df["sma_slow"].iloc[-1]:
                signal = "Sell"

            # MACD and ADX signals
            if df["macd"].iloc[-2] < df["macd_signal"].iloc[-2] and df["macd"].iloc[-1] > df["macd_signal"].iloc[-1] and df["adx"].iloc[-1] > 25 and df["cci"].iloc[-1] > 100:
                signal = "Buy"
            elif df["macd"].iloc[-2] > df["macd_signal"].iloc[-2] and df["macd"].iloc[-1] < df["macd_signal"].iloc[-1] and df["adx"].iloc[-1] > 25 and df["cci"].iloc[-1] < -100:
                signal = "Sell"

            # Bollinger Bands and Ichimoku Cloud signals
            if (
                df["bollinger_hband"].iloc[-1] > df["close"].iloc[-1] > df["bollinger_mavg"].iloc[-1]
                and df["aroon_up"].iloc[-1] > 70
                and df["aroon_down"].iloc[-1] < 30
                and df["ichimoku_a"].iloc[-1] > df["ichimoku_b"].iloc[-1]
            ):
                signal = "Buy"
            elif (
                df["bollinger_lband"].iloc[-1] < df["close"].iloc[-1] < df["bollinger_mavg"].iloc[-1]
                and df["aroon_up"].iloc[-1] < 30
                and df["aroon_down"].iloc[-1] > 70
                and df["ichimoku_a"].iloc[-1] < df["ichimoku_b"].iloc[-1]
            ):
                signal = "Sell"

            support_level, resistance_level = self.calculate_support_and_resistance(df)

            # Risk Management
            stop_loss, take_profit = self.calculate_stop_loss_take_profit(signal, current_price, support_level, resistance_level)
            
            metrics = {
                "price": current_price,
                "stop_loss": stop_loss,
                "take_profit": take_profit,
                "support": support_level,
                "resistance": resistance_level
            }

            logger.info(f"Generated signal for {pair} ({timeframe}): {signal}")
            return signal, metrics
        except Exception as e:
            logger.error(f"Error generating signal for {pair} ({timeframe}): {str(e)}")
            return "Neutral", {}

    def calculate_support_and_resistance(self, df: pd.DataFrame) -> Tuple[float, float]:
        try:
            # Calculate support and resistance levels as the lowest and highest of the last `num_candles`.
            support_level = df["low"].tail(self.num_candles).min()
            resistance_level = df["high"].tail(self.num_candles).max()
            return support_level, resistance_level
        except Exception as e:
            logger.error(f"Error calculating support and resistance: {str(e)}")
            return df["low"].iloc[-1], df["high"].iloc[-1]  # Fall back to last row values in case of error.

    def calculate_stop_loss_take_profit(self, signal: str, current_price: float, support: float, resistance: float) -> Tuple[float, float]:
        if signal in ["Buy", "Strong Buy"]:
            stop_loss = support
            take_profit = resistance
        elif signal in ["Sell", "Strong Sell"]:
            stop_loss = resistance
            take_profit = support
        else:
            stop_loss = support
            take_profit = resistance

        return stop_loss, take_profit

    async def save_signal(self, pair: str, timeframe: str, signal: str, metrics: Dict[str, float]):
        try:
            Signal.objects.create(
                symbol=pair,
                timestamp=pd.Timestamp.now(),
                timeframe=timeframe,
                signal_type=signal,
                price=metrics.get("price"),
                stop_loss=metrics.get("stop_loss"),
                take_profit=metrics.get("take_profit"),
            )
            logger.info(f"Signal for {pair} {timeframe} saved to the database.")
        except Exception as e:
            logger.error(f"Error saving signal for {pair} {timeframe}: {e}")

    async def run_bot(self):
        while True:
            logger.info("Starting bot cycle...")
            start_time = time.time()

            tasks = []
            symbols = TradingPair.objects.values_list("symbol", flat=True)

            for symbol in symbols:
                for timeframe in self.timeframes:
                    task = self.process_symbol(symbol, timeframe)
                    tasks.append(task)

            await asyncio.gather(*tasks)

            end_time = time.time()
            execution_time = end_time - start_time
            sleep_time = max(60 - execution_time, 0)
            logger.info(f"Bot cycle complete. Sleeping for {sleep_time} seconds.")
            await asyncio.sleep(sleep_time)

    async def process_symbol(self, symbol: str, timeframe: str):
        signal, metrics = await self.generate_signal(symbol, timeframe)
        await self.save_signal(symbol, timeframe, signal, metrics)

if __name__ == "__main__":
    bot = CryptoBot()
    try:
        asyncio.run(bot.run_bot())
    except KeyboardInterrupt:
        logger.info("Stopping bot...")

Above is my actual logic code that I will integrate with the website, can you now update for me models.py, views.py, urls.py, settings.py and all other django files that need to be updated using above information.