Data Fetching takes too much time, make it faster: import os
import datetime
import time
from flask import Flask, render_template, request, redirect, url_for, session
from flask_session import Session  # Import Flask-Session
from binance.client import Client
from binance.exceptions import BinanceAPIException
import pandas as pd
import plotly.graph_objs as go
from plotly.offline import plot

# Initialize the Flask application
app = Flask(__name__)

# Set a secret key for sessions
app.secret_key = 'your_secret_key_here'  # Replace with your secret key

# Configure server-side sessions
app.config['SESSION_TYPE'] = 'filesystem'  # Use filesystem-based sessions
app.config['SESSION_PERMANENT'] = False
app.config['SESSION_FILE_DIR'] = os.path.join(app.root_path, 'flask_session')
Session(app)  # Initialize the extension

# Configure Binance Client (No API key needed for public data)
client = Client()

# Default Symbols and Timeframes
default_symbols = [
    'BTCUSDT', 'ETHUSDT', 'XRPUSDT', 'DOGEUSDT', 'SOLUSDT',
    'BNBUSDT', 'SUIUSDT', 'CKBUSDT', 'FTMUSDT', 'RUNEUSDT',
    'UNIUSDT', 'AAVEUSDT', 'BCHUSDT', 'KEYUSDT', '1INCHUSDT',
    'NEARUSDT', 'ADAUSDT', 'ATOMUSDT', 'LTCUSDT', 'STXUSDT',
    'RENDERUSDT', 'FETUSDT', 'CRVUSDT', 'RSRUSDT', 'MINAUSDT',
    'WOOUSDT', 'WIFUSDT', 'CELOUSDT', 'CFXUSDT', 'GALAUSDT',
    'INJUSDT', 'IOTXUSDT', 'DYDXUSDT', 'XECUSDT', 'GASUSDT',
    'CAKEUSDT', 'SCUSDT', 'DOTUSDT', 'AVAXUSDT', 'VETUSDT',
    'ICPUSDT', 'FILUSDT', 'ROSEUSDT', 'ARBUSDT', 'RAYUSDT',
    'OPUSDT', 'LPTUSDT', 'TUSDT', 'MASKUSDT', 'BLURUSDT',
    'ZECUSDT', 'ARKMUSDT', 'ZRXUSDT', 'TIAUSDT', 'KSMUSDT',
    'EOSUSDT', 'BOMEUSDT', 'ETCUSDT', 'GRTUSDT', 'ZKUSDT',
    'MANAUSDT', 'JSTUSDT', 'TWTUSDT', 'ENJUSDT', 'JUPUSDT',
    'ARUSDT', 'BONKUSDT', 'SANDUSDT', 'ENSUSDT', 'AXSUSDT',
    'KAVAUSDT', 'JASMYUSDT', 'DASHUSDT', 'AXLUSDT', 'ORDIUSDT',
    'LUNAUSDT', 'PYTHUSDT', '1000SATSUSDT', 'ZROUSDT', 'NOTUSDT',
    'FLOKIUSDT', 'BEAMXUSDT', 'DEXEUSDT', 'COMPUSDT', 'SHIBUSDT'
]


default_timeframes = ['30m', '1h', '2h', '4h', '1d']

# Rate limit parameters
API_RATE_LIMIT = 150  # Max 150 requests per minute
REQUEST_INTERVAL = 60 / API_RATE_LIMIT  # Interval between requests

# Function to fetch historical candlestick data
def get_historical_klines(symbol, interval, limit=150):
    try:
        # Sleep to respect the API rate limit
        time.sleep(REQUEST_INTERVAL)
        klines = client.get_klines(symbol=symbol, interval=interval, limit=limit)
        df = pd.DataFrame(klines, columns=[
            'timestamp', 'open', 'high', 'low', 'close', 'volume',
            'close_time', 'quote_asset_volume', 'number_of_trades',
            'taker_buy_base', 'taker_buy_quote', 'ignore'
        ])
        df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms')
        df[['open', 'high', 'low', 'close']] = df[['open', 'high', 'low', 'close']].astype(float)
        return df
    except BinanceAPIException as e:
        print(f"Error fetching data for {symbol}: {e}")
        return None

# Function to identify demand zones (RBR and DBR patterns) with variable base lengths and overlapping patterns
def identify_demand_zones(df):
    demand_zones = []
    max_base_length = 6  # Maximum number of base candles
    min_base_length = 1  # Minimum number of base candles

    for i in range(1, len(df) - min_base_length - 1):
        # Check if the prior candle is a rally (bullish candle)
        if df['close'].iloc[i - 1] <= df['open'].iloc[i - 1]:
            continue  # Not a rally, skip to next index

        # Try base lengths from 1 to 6
        for base_length in range(min_base_length, min(max_base_length + 1, len(df) - i - 1)):
            base_end_index = i + base_length - 1
            next_candle_index = base_end_index + 1

            if next_candle_index >= len(df):
                break  # Reached the end of the DataFrame

            # Check if the next candle is a rally (bullish candle)
            if df['close'].iloc[next_candle_index] <= df['open'].iloc[next_candle_index]:
                continue  # Next candle is not a rally

            # Check base candles criteria
            base_candles = df.iloc[i:base_end_index + 1]
            base_bodies = abs(base_candles['close'] - base_candles['open'])

            # Ensure no base candle body is equal to or larger than rally candles
            body_rally_before = abs(df['close'].iloc[i - 1] - df['open'].iloc[i - 1])
            body_rally_after = abs(df['close'].iloc[next_candle_index] - df['open'].iloc[next_candle_index])

            valid_base = True
            for base_body in base_bodies:
                if (base_body >= body_rally_before) or (base_body >= body_rally_after):
                    valid_base = False
                    break  # Base candle body is not strictly smaller

            if not valid_base:
                continue  # Base candles do not meet criteria

            # If all base candles are valid, identify the zone
            lower_bound = base_candles['low'].min()
            upper_bound = base_candles[['open', 'close']].max().max()

            zone = {
                'start_time': df['timestamp'].iloc[i - 1],  # Start from the rally before the base
                'end_time': df['timestamp'].iloc[next_candle_index],  # End at the rally after the base
                'upper_bound': upper_bound,
                'lower_bound': lower_bound,
                'type': 'RBR',
                'formation_end_index': next_candle_index  # For checking re-entries after formation
            }

            # Avoid duplicate zones
            if zone not in demand_zones:
                demand_zones.append(zone)

    return demand_zones

# Function to identify supply zones (DBD and RBD patterns) with variable base lengths and overlapping patterns
def identify_supply_zones(df):
    supply_zones = []
    max_base_length = 6  # Maximum number of base candles
    min_base_length = 1  # Minimum number of base candles

    for i in range(1, len(df) - min_base_length - 1):
        # Check if the prior candle is a drop (bearish candle)
        if df['close'].iloc[i - 1] >= df['open'].iloc[i - 1]:
            continue  # Not a drop, skip to next index

        # Try base lengths from 1 to 6
        for base_length in range(min_base_length, min(max_base_length + 1, len(df) - i - 1)):
            base_end_index = i + base_length - 1
            next_candle_index = base_end_index + 1

            if next_candle_index >= len(df):
                break  # Reached the end of the DataFrame

            # Check if the next candle is a drop (bearish candle)
            if df['close'].iloc[next_candle_index] >= df['open'].iloc[next_candle_index]:
                continue  # Next candle is not a drop

            # Check base candles criteria
            base_candles = df.iloc[i:base_end_index + 1]
            base_bodies = abs(base_candles['close'] - base_candles['open'])

            # Ensure no base candle body is equal to or larger than drop candles
            body_drop_before = abs(df['close'].iloc[i - 1] - df['open'].iloc[i - 1])
            body_drop_after = abs(df['close'].iloc[next_candle_index] - df['open'].iloc[next_candle_index])

            valid_base = True
            for base_body in base_bodies:
                if (base_body >= body_drop_before) or (base_body >= body_drop_after):
                    valid_base = False
                    break  # Base candle body is not strictly smaller

            if not valid_base:
                continue  # Base candles do not meet criteria

            # If all base candles are valid, identify the zone
            upper_bound = base_candles['high'].max()
            lower_bound = base_candles[['open', 'close']].min().min()

            zone = {
                'start_time': df['timestamp'].iloc[i - 1],  # Start from the drop before the base
                'end_time': df['timestamp'].iloc[next_candle_index],  # End at the drop after the base
                'upper_bound': upper_bound,
                'lower_bound': lower_bound,
                'type': 'DBD',
                'formation_end_index': next_candle_index  # For checking re-entries after formation
            }

            # Avoid duplicate zones
            if zone not in supply_zones:
                supply_zones.append(zone)

    return supply_zones

# Function to check if the zone is fresh, tested, or violated
def check_zone_freshness(df, zone):
    # Start checking from the candle after the formation ends
    formation_end_idx = zone['formation_end_index']
    zone_df = df.iloc[formation_end_idx + 1:]

    if zone_df.empty:
        zone['status'] = 'Fresh'
        return zone

    # Check if price has re-entered the zone after the formation
    reentries = zone_df[
        (zone_df['high'] >= zone['lower_bound']) & (zone_df['low'] <= zone['upper_bound'])
    ]

    if reentries.empty:
        zone['status'] = 'Fresh'
    else:
        # For supply zone
        if zone['type'] in ['DBD', 'RBD']:
            # Check if the zone is violated (price closed above upper bound)
            violations = reentries[reentries['close'] > zone['upper_bound']]
            if not violations.empty:
                zone['status'] = 'Violated'
            else:
                zone['status'] = 'Tested'
        # For demand zone
        elif zone['type'] in ['RBR', 'DBR']:
            # Check if the zone is violated (price closed below lower bound)
            violations = reentries[reentries['close'] < zone['lower_bound']]
            if not violations.empty:
                zone['status'] = 'Violated'
            else:
                zone['status'] = 'Tested'
        else:
            zone['status'] = 'Unknown'
    return zone

# Function to plot the candlestick chart with supply and demand zones
def plot_chart(df, zones, symbol, interval):
    fig = go.Figure(data=[go.Candlestick(
        x=df['timestamp'],
        open=df['open'],
        high=df['high'],
        low=df['low'],
        close=df['close'],
        name='Candlesticks'
    )])

    # Add zones
    for zone in zones:
        if zone['status'] == 'Fresh':
            if zone['type'] in ['DBD', 'RBD']:
                color = 'red'  # Supply zone color
            else:
                color = 'green'  # Demand zone color
            opacity = 0.2
        elif zone['status'] == 'Tested':
            if zone['type'] in ['DBD', 'RBD']:
                color = 'orange'  # Supply zone tested color
            else:
                color = 'lightgreen'  # Demand zone tested color
            opacity = 0.2
        else:
            # Skip violated zones
            continue

        fig.add_shape(type='rect',
                      x0=zone['start_time'], y0=zone['lower_bound'],
                      x1=df['timestamp'].iloc[-1], y1=zone['upper_bound'],
                      fillcolor=color, opacity=opacity,
                      line=dict(width=0))

        fig.add_annotation(
            x=zone['start_time'],
            y=zone['upper_bound'],
            xref="x",
            yref="y",
            text=f"{zone['type']} ({zone['status']})",
            showarrow=True,
            arrowhead=1,
            font=dict(color=color)
        )

    fig.update_layout(title=f"{symbol} - {interval} Chart with Supply and Demand Zones",
                      xaxis_title='Time',
                      yaxis_title='Price',
                      xaxis_rangeslider_visible=False)

    return plot(fig, output_type='div')

# Define the Flask routes
@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        # Get user inputs for symbols and timeframes
        symbol_input = request.form.get('symbols', '')
        interval_input = request.form.getlist('intervals')

        # Process the symbols input into a list
        user_symbols = [s.strip().upper() for s in symbol_input.split(',') if s.strip()]
        # Use default symbols if none provided
        if not user_symbols:
            user_symbols = default_symbols

        # Process the intervals input
        user_timeframes = interval_input or default_timeframes

        # Get user inputs for zone counts and inclusion of tested zones
        try:
            fresh_supply_count = int(request.form.get('fresh_supply_count', '2'))
            fresh_demand_count = int(request.form.get('fresh_demand_count', '2'))
            tested_supply_count = int(request.form.get('tested_supply_count', '1'))
            tested_demand_count = int(request.form.get('tested_demand_count', '1'))
        except ValueError:
            fresh_supply_count = 2
            fresh_demand_count = 2
            tested_supply_count = 1
            tested_demand_count = 1

        include_tested_supply = request.form.get('include_tested_supply') == 'on'
        include_tested_demand = request.form.get('include_tested_demand') == 'on'

        # Save to session
        session['symbols'] = user_symbols
        session['timeframes'] = user_timeframes
        session['fresh_supply_count'] = fresh_supply_count
        session['fresh_demand_count'] = fresh_demand_count
        session['tested_supply_count'] = tested_supply_count
        session['tested_demand_count'] = tested_demand_count
        session['include_tested_supply'] = include_tested_supply
        session['include_tested_demand'] = include_tested_demand

        # Now proceed to scan the selected symbols and intervals
        results = []  # To store the findings
        total_combinations = len(user_symbols) * len(user_timeframes)
        processed = 0
        for symbol in user_symbols:
            for interval in user_timeframes:
                # Fetch data
                df = get_historical_klines(symbol, interval, limit=150)
                if df is not None:
                    # Identify supply and demand zones
                    supply_zones = identify_supply_zones(df)
                    demand_zones = identify_demand_zones(df)
                    all_zones = supply_zones + demand_zones

                    # Check zone freshness
                    all_zones = [check_zone_freshness(df, zone) for zone in all_zones if zone is not None]

                    # Filter out violated zones
                    valid_zones = [zone for zone in all_zones if zone['status'] != 'Violated']

                    # Proceed only if there are valid zones
                    if valid_zones:
                        # Sort zones by start_time in descending order to get the most recent zones first
                        valid_zones.sort(key=lambda x: x['start_time'], reverse=True)

                        # Get supply zones
                        supply_zones_list = [zone for zone in valid_zones if zone['type'] in ['DBD', 'RBD']]

                        # Get demand zones
                        demand_zones_list = [zone for zone in valid_zones if zone['type'] in ['RBR', 'DBR']]

                        # Apply user's preferences for supply zones
                        fresh_supply_zones = [zone for zone in supply_zones_list if zone['status'] == 'Fresh'][:fresh_supply_count]

                        if include_tested_supply:
                            tested_supply_zones = [zone for zone in supply_zones_list if zone['status'] == 'Tested'][:tested_supply_count]
                        else:
                            tested_supply_zones = []

                        # Apply user's preferences for demand zones
                        fresh_demand_zones = [zone for zone in demand_zones_list if zone['status'] == 'Fresh'][:fresh_demand_count]

                        if include_tested_demand:
                            tested_demand_zones = [zone for zone in demand_zones_list if zone['status'] == 'Tested'][:tested_demand_count]
                        else:
                            tested_demand_zones = []

                        # Combine the selected zones
                        selected_zones = fresh_supply_zones + tested_supply_zones + fresh_demand_zones + tested_demand_zones

                        # Append results for symbol and interval
                        results.append({
                            'symbol': symbol,
                            'interval': interval,
                            'zones': selected_zones
                        })
                else:
                    print(f"Data not available for {symbol} on timeframe {interval}")
                processed += 1
                print(f"Processed {processed}/{total_combinations}")
        # After processing, group the results by symbol
        grouped_results = {}
        for result in results:
            symbol = result['symbol']
            if symbol not in grouped_results:
                grouped_results[symbol] = []
            grouped_results[symbol].append({
                'interval': result['interval'],
                'zones': result['zones']
            })
        session['grouped_results'] = grouped_results  # Save results in session
        return redirect(url_for('results'))
    else:
        return render_template('index.html', default_symbols=', '.join(default_symbols),
                               default_timeframes=default_timeframes,
                               current_year=datetime.datetime.now().year)

@app.route('/results')
def results():
    grouped_results = session.get('grouped_results', {})
    return render_template('results.html', grouped_results=grouped_results,
                           current_year=datetime.datetime.now().year)

@app.route('/charts/<symbol>', methods=['GET'])
def symbol_charts(symbol):
    # Retrieve intervals from session
    user_timeframes = session.get('timeframes', default_timeframes)
    # Retrieve user preferences
    fresh_supply_count = session.get('fresh_supply_count', 2)
    fresh_demand_count = session.get('fresh_demand_count', 2)
    tested_supply_count = session.get('tested_supply_count', 1)
    tested_demand_count = session.get('tested_demand_count', 1)
    include_tested_supply = session.get('include_tested_supply', True)
    include_tested_demand = session.get('include_tested_demand', True)

    # Generate charts for all intervals for this symbol
    charts = []
    for interval in user_timeframes:
        df = get_historical_klines(symbol, interval, limit=150)
        if df is not None:
            # Identify zones as before
            supply_zones = identify_supply_zones(df)
            demand_zones = identify_demand_zones(df)
            all_zones = supply_zones + demand_zones

            # Check zone freshness
            all_zones = [check_zone_freshness(df, zone) for zone in all_zones if zone is not None]

            # Filter out violated zones
            valid_zones = [zone for zone in all_zones if zone['status'] != 'Violated']

            # Sort zones by start_time in descending order to get the most recent zones first
            valid_zones.sort(key=lambda x: x['start_time'], reverse=True)

            # Get supply zones
            supply_zones_list = [zone for zone in valid_zones if zone['type'] in ['DBD', 'RBD']]

            # Get demand zones
            demand_zones_list = [zone for zone in valid_zones if zone['type'] in ['RBR', 'DBR']]

            # Apply user's preferences for supply zones
            fresh_supply_zones = [zone for zone in supply_zones_list if zone['status'] == 'Fresh'][:fresh_supply_count]

            if include_tested_supply:
                tested_supply_zones = [zone for zone in supply_zones_list if zone['status'] == 'Tested'][:tested_supply_count]
            else:
                tested_supply_zones = []

            # Apply user's preferences for demand zones
            fresh_demand_zones = [zone for zone in demand_zones_list if zone['status'] == 'Fresh'][:fresh_demand_count]

            if include_tested_demand:
                tested_demand_zones = [zone for zone in demand_zones_list if zone['status'] == 'Tested'][:tested_demand_count]
            else:
                tested_demand_zones = []

            # Combine the selected zones
            selected_zones = fresh_supply_zones + tested_supply_zones + fresh_demand_zones + tested_demand_zones

            chart_div = plot_chart(df, selected_zones, symbol, interval)
            charts.append({'interval': interval, 'chart_div': chart_div})
        else:
            print(f"Data not available for {symbol} on timeframe {interval}")
    return render_template('symbol_charts.html', symbol=symbol, charts=charts,
                           current_year=datetime.datetime.now().year)

# Run the Flask application
if __name__ == '__main__':
    app.run(debug=True)