Remember my code: import redis
import json
import logging
import datetime
import pytz
import re
import os
import asyncio
import aiohttp
from datetime import timezone, timedelta
from concurrent.futures import ThreadPoolExecutor
from functools import partial
from urllib.parse import urlparse, parse_qs

# Configuration parameters for moving stop loss, selling based on QQQ below or above EMA
Moving_Stop_Loss = 'Y'  # Default value
Max_Buys = 4  # Global variable to limit the number of buys per day
daily_buy_limit = 14
Daily_RSI = 'N'
market_close_time = datetime.time(hour=15, minute=59)
market_close_time_for_scalp = datetime.time(hour=15, minute=59, second=50)  # 10 seconds before close
market_open_time = datetime.time(hour=9, minute=34)
daily_buy_count = {}

# Scalp trade parameters
Scalp_Profit_Taker = 0.008  # 0.3% = 0.003
Scalp_Stop_Loss = 0.015  # 0.3%

# Setup logging
# Delete the existing log file if it exists
if os.path.exists('log_simulation.txt'):
    os.remove('log_simulation.txt')

logging.basicConfig(filename='log_simulation.txt', level=logging.INFO,
                    format='%(asctime)s - %(levelname)s - %(message)s')

# Connect to Redis database
redis_host = '127.0.0.1'
redis_port = 6379
redis_db = redis.Redis(host=redis_host, port=redis_port)

# Define the node name
node_name = 'backtesting_tool'

# Part 1: Delete `backtesting_tool:*` keys before starting the backtesting logic
def delete_backtesting_tool_keys():
    keys = redis_db.keys(pattern='backtesting_tool:*')
    if keys:
        redis_db.delete(*keys)
        print(f"Deleted {len(keys)} keys matching the pattern 'backtesting_tool:*'")

# Part 2: Copy keys from `backtesting_tool2:*` to `backtesting_tool:`
def copy_redis_keys():
    # Connect to the Redis instance
    r = redis.Redis(host='localhost', port=6379, db=0)

    # Define the source and destination key patterns
    src_pattern = b'backtesting_tool3:*'
    dst_pattern = 'backtesting_tool:'

    # Get a list of keys matching the source pattern
    src_keys = r.keys(pattern=src_pattern)

    # Iterate over the source keys and copy their values to the destination keys
    for src_key in src_keys:
        # Get the data type of the source key
        src_type = r.type(src_key).decode('utf-8')

        if src_type == 'hash':
            # Get the hash fields and values of the source key
            src_hash = r.hgetall(src_key)

            # Construct the destination key name
            dst_key = dst_pattern + src_key.decode('utf-8')[len('backtesting_tool3:'):]

            # Copy the hash to the destination key
            r.hmset(dst_key, src_hash)
        else:
            # Get the value of the source key
            src_value = r.get(src_key)

            # Construct the destination key name
            dst_key = dst_pattern + src_key.decode('utf-8')[len('backtesting_tool3'):]

            # Copy the value to the destination key
            r.set(dst_key, src_value)

    print(f'Copied {len(src_keys)} keys from backtesting_tool2 to backtesting_tool')

# Run the copy logic before proceeding with backtesting
delete_backtesting_tool_keys()  # Delete old keys
copy_redis_keys()  # Copy keys from `backtesting_tool2` to `backtesting_tool`

# Global caches
api_cache = {}
rsi_data_cache = {}
intraday_data_cache = {}
ema8_data_cache = {}
redis_fields_cache = {}

# Function to retrieve position size from Site_Configuration node
def get_position_size():
    position_size = redis_db.hget('Site_Configuration', 'Position size')
    if position_size:
        return float(position_size.decode('utf-8'))
    else:
        logging.error("Position size not found in Site_Configuration.")
        return None

async def fetch_intraday_data(session, ticker, start_date, end_date, multiplier=1, timespan='second'):
    start_date_str = start_date.strftime('%Y-%m-%d')
    end_date_str = end_date.strftime('%Y-%m-%d')
    url = f"https://api.polygon.io/v2/aggs/ticker/{ticker}/range/{multiplier}/{timespan}/{start_date_str}/{end_date_str}?adjusted=true&sort=asc&limit=50000&apiKey=YOUR_API_KEY"  # Replace with your actual API key
    logging.info(f"Fetching {timespan} data for {ticker} from {start_date_str} to {end_date_str}")
    async with session.get(url) as response:
        if response.status == 200:
            try:
                return await response.json()
            except json.decoder.JSONDecodeError:
                logging.error(f"Error decoding JSON response for {ticker} from {start_date_str} to {end_date_str}: {await response.text()}")
                return None
        else:
            logging.error(f"API request failed with status code {response.status} for {ticker} from {start_date_str} to {end_date_str}")
            return None

async def fetch_rsi_data(session, ticker, start_date, end_date, timespan='day', window=12, series_type='close'):
    start_date_str = start_date.strftime('%Y-%m-%d')
    end_date_str = end_date.strftime('%Y-%m-%d')
    url = f"https://api.polygon.io/v1/indicators/rsi/{ticker}?timestamp.gte={start_date_str}&timespan={timespan}&adjusted=true&window={window}&series_type={series_type}&order=asc&limit=5000&apiKey=YOUR_API_KEY"
    logging.info(f"Fetching RSI data for {ticker}: {url}")
    async with session.get(url) as response:
        if response.status == 200:
            try:
                return await response.json()
            except json.decoder.JSONDecodeError:
                logging.error(f"Error decoding JSON response for {ticker} from {start_date_str} to {end_date_str}: {await response.text()}")
                return None
        else:
            logging.error(f"API request failed with status code {response.status} for {ticker} from {start_date_str} to {end_date_str}")
            return None

async def fetch_ema_data(session, ticker, start_date, end_date, window=8):
    start_date_str = start_date.strftime('%Y-%m-%d')
    end_date_str = end_date.strftime('%Y-%m-%d')
    url = f"https://api.polygon.io/v1/indicators/ema/{ticker}?timestamp.gte={start_date_str}&timespan=day&adjusted=true&window={window}&series_type=close&order=asc&limit=5000&apiKey=YOUR_API_KEY"
    logging.info(f"Fetching EMA data for {ticker}: {url}")
    async with session.get(url) as response:
        if response.status == 200:
            try:
                return await response.json()
            except json.decoder.JSONDecodeError:
                logging.error(f"Error decoding JSON response for {ticker} from {start_date_str} to {end_date_str}: {await response.text()}")
                return None
        else:
            logging.error(f"API request failed with status code {response.status} for {ticker} from {start_date_str} to {end_date_str}")
            return None

async def fetch_all_data(ticker_date_ranges, rsi_dates):
    async with aiohttp.ClientSession() as session:
        tasks = []
        intraday_tasks = []

        # Intraday data tasks
        for ticker, date_info in ticker_date_ranges.items():
            buy_dates = date_info['buy_dates']
            end_date = date_info['end_date']
            earliest_buy_date = min(buy_dates)

            # Fetch 1-second data for each buy date
            for buy_date in buy_dates:
                task_1s = fetch_intraday_data(session, ticker, buy_date, buy_date, multiplier=1, timespan='second')
                intraday_tasks.append((ticker, buy_date, '1_second', task_1s))

            # Fetch 1-minute data from the day after earliest buy date to end_date
            if earliest_buy_date < end_date:
                start_date = earliest_buy_date + datetime.timedelta(days=1)
                task_1m = fetch_intraday_data(session, ticker, start_date, end_date, multiplier=1, timespan='minute')
                intraday_tasks.append((ticker, None, '1_minute', task_1m))

        # RSI data tasks for QQQ
        rsi_tasks = []
        rsi_tasks.append(fetch_rsi_data(session, 'QQQ', rsi_dates['start_date'], rsi_dates['end_date'], timespan='day'))

        # EMA data tasks
        ema_tasks = []
        for ticker in ticker_date_ranges.keys():
            ema_tasks.append(fetch_ema_data(session, ticker, ticker_date_ranges[ticker]['start_date'], ticker_date_ranges[ticker]['end_date']))

        # Gather all tasks
        tasks = [task for _, _, _, task in intraday_tasks] + rsi_tasks + ema_tasks
        results = await asyncio.gather(*tasks)

        # Build data caches
        intraday_data = {}
        intraday_results_count = len(intraday_tasks)
        for (ticker, buy_date, interval, _), result in zip(intraday_tasks, results[:intraday_results_count]):
            if ticker not in intraday_data:
                intraday_data[ticker] = {}
            if buy_date:
                # For 1-second data per buy date
                if 'buy_dates' not in intraday_data[ticker]:
                    intraday_data[ticker]['buy_dates'] = {}
                intraday_data[ticker]['buy_dates'][buy_date] = result
            else:
                # For 1-minute data
                intraday_data[ticker]['1_minute'] = result

        rsi_data_results = results[intraday_results_count:intraday_results_count+len(rsi_tasks)]
        ema_data_results = results[-len(ema_tasks):]

        rsi_data = {}
        for ticker, data in zip(['QQQ'], rsi_data_results):
            rsi_data[ticker] = data

        ema_data = {}
        for ticker, data in zip(ticker_date_ranges.keys(), ema_data_results):
            ema_data[ticker] = data

    return intraday_data, rsi_data, ema_data

def get_rsi_value_for_previous_available_day(start_date_str, rsi_values_by_date):
    # start_date_str is a string in 'YYYY-MM-DD' format
    date = datetime.datetime.strptime(start_date_str, '%Y-%m-%d')
    max_lookback_days = 10  # You can adjust this as needed
    for i in range(1, max_lookback_days + 1):
        previous_date = date - datetime.timedelta(days=i)
        previous_date_str = previous_date.strftime('%Y-%m-%d')
        rsi_value = rsi_values_by_date.get(previous_date_str)
        if rsi_value is not None:
            return rsi_value, previous_date_str
    return None, None

def calculate_result_for_key(key, intraday_data, rsi_data, ema_data):
    key_decoded = key.decode('utf-8')
    parts = key_decoded.split(":")
    if len(parts) != 3:
        logging.error(f"Invalid key format: {key_decoded}")
        return

    ticker = parts[2]
    logging.info(f"Calculating results for key: {key_decoded}")
    logging.info(f"Calculating results for ticker: {ticker}")

    # Retrieve data from Redis or cache
    if key_decoded in redis_fields_cache:
        fields = redis_fields_cache[key_decoded]
    else:
        fields_raw = redis_db.hgetall(key)
        if not fields_raw:
            logging.error(f"Key {key_decoded} has no fields.")
            return
        fields = {k.decode('utf-8'): v.decode('utf-8') for k, v in fields_raw.items()}
        redis_fields_cache[key_decoded] = fields

    # Check if the key is a hash type
    if 'Ticker' in fields:
        # Greenband or ORH as buy price
        check_green_band = fields.get('ORHHigh')
        if check_green_band is None:
            logging.warning(f"Warning: Key '{key_decoded}' or field 'ORHHigh' not found in Redis.")
            return  # Exit the function or handle as needed
        green_band = float(fields['ORHHigh'])
        buy_price = green_band
        fields['BuyPrice'] = buy_price

        # Use pipeline to batch Redis commands
        pipeline = redis_db.pipeline()

        # Batch hset commands
        pipeline.hset(key, 'BuyPrice', buy_price)

        # Extract necessary data from fields
        buy_date_str = fields['BuyDate']
        fix_stop = 0.97 * buy_price  # Fixed SL 3%
        fields['StopLoss'] = fix_stop
        pipeline.hset(key, 'StopLoss', fix_stop)
        stop_loss = float(fields['StopLoss'])
        trade_type = fields.get('TradeType')
        industry_rs_percentile_value = fields.get('Industry_RSPercentile')

        # Validate Industry_RSPercentile
        if industry_rs_percentile_value is not None:
            Industry_RSPercentile = float(industry_rs_percentile_value)
        else:
            logging.warning(f"Industry_RSPercentile not found for key: {key_decoded}")
            Industry_RSPercentile = None  # Or set a default value if needed

        risk_reward = ((buy_price - stop_loss) / buy_price) * 100
        buy_date_dt = datetime.datetime.strptime(buy_date_str, '%Y%m%d')
        formatted_buy_date = buy_date_dt.strftime('%Y-%m-%d %H:%M:%S')
        logging.info(f"Ticker: {ticker}, Buy Price: {buy_price}, Buy Date: {formatted_buy_date}")

        start_date = buy_date_dt
        end_date = datetime.datetime.now()
        initial_date = start_date

        is_risk_reward_2_reached = False
        is_risk_reward_5_reached = False
        stop_loss_hit = False
        # Initialize flag to track buy price reached and timestamp
        buy_price_reached = False

        # Retrieve position size
        position_size = get_position_size()
        if position_size is None:
            logging.error("Position size is not set. Exiting calculation.")
            return
        # Initialize variable to store previous day
        previous_day = None
        # Define the US Eastern Time zone
        eastern = pytz.timezone('US/Eastern')

        # Extract intraday data for the ticker and buy date
        ticker_intraday_data = intraday_data.get(ticker)
        if not ticker_intraday_data:
            logging.error(f"No intraday data for ticker {ticker}")
            return

        # Get 1-second data for buy date
        ticker_intraday_data_buy_dates = ticker_intraday_data.get('buy_dates')
        if not ticker_intraday_data_buy_dates:
            logging.error(f"No intraday data for ticker {ticker} buy dates")
            return

        buy_date_data = ticker_intraday_data_buy_dates.get(buy_date_dt)
        if not buy_date_data:
            logging.error(f"No intraday data for ticker {ticker} on buy date {buy_date_dt}")
            return

        ticker_intraday_data_1s = buy_date_data
        if ticker_intraday_data_1s is None or 'results' not in ticker_intraday_data_1s:
            logging.error(f"No 1-second intraday data for ticker {ticker} on buy date {buy_date_dt}")
            return
        # Get 1-minute data for subsequent days
        ticker_intraday_data_1m = ticker_intraday_data.get('1_minute')
        if ticker_intraday_data_1m is None or 'results' not in ticker_intraday_data_1m:
            logging.error(f"No 1-minute intraday data for ticker {ticker}")
            return

        # Build rsi_values_by_date once
        if Daily_RSI == 'Y':
            rsi_data_for_date = rsi_data.get('QQQ')
            if rsi_data_for_date and 'results' in rsi_data_for_date and 'values' in rsi_data_for_date['results']:
                rsi_data_values = rsi_data_for_date['results']['values']
                # Build a mapping from date string to RSI value
                rsi_values_by_date = {}
                for rsi_entry in rsi_data_values:
                    timestamp_ms = rsi_entry['timestamp']
                    timestamp_dt = datetime.datetime.fromtimestamp(timestamp_ms / 1000)
                    date_str = timestamp_dt.strftime('%Y-%m-%d')
                    rsi_values_by_date[date_str] = rsi_entry['value']
            else:
                logging.error(f"No RSI data for QQQ")
                return

        # Process the buy date using 1-second data
        formatted_buy_date = buy_date_dt.strftime('%Y-%m-%d')

        # Organize 1-second data for the buy date
        intraday_results_1s = ticker_intraday_data_1s['results']
        intraday_results_by_date_1s = {}
        for result in intraday_results_1s:
            timestamp = datetime.datetime.fromtimestamp(result['t']/1000)
            date_str = timestamp.strftime('%Y-%m-%d')
            if date_str not in intraday_results_by_date_1s:
                intraday_results_by_date_1s[date_str] = []
            intraday_results_by_date_1s[date_str].append(result)

        # Get data for the buy date
        date_intraday_data = intraday_results_by_date_1s.get(formatted_buy_date)
        if not date_intraday_data:
            logging.warning(f"No intraday data for {ticker} on {formatted_buy_date}")
            return

        prices = [float(result['c']) for result in date_intraday_data]
        price_timestamps = [result['t'] for result in date_intraday_data]

        filtered_data = [
            ((datetime.datetime.fromtimestamp(ts / 1000).astimezone(eastern).strftime('%H:%M:%S.%f')[:-3]), price)
            for ts, price in zip(price_timestamps, prices)
            if market_close_time > datetime.datetime.fromtimestamp(ts / 1000).astimezone(eastern).time() > market_open_time
        ]

        logging.info(f"Filtered data for buy date {formatted_buy_date}: {len(filtered_data)} data points")

        # Variables for scalp trade
        scalp_trade_active = False
        scalp_stop_loss_hit = False
        scalp_profit_taker_hit = False
        scalp_sell_price = None
        scalp_sell_time = None

        # Iterate over prices for the buy date
        for us_intraday_timestamp, price in filtered_data:
            time_obj = datetime.datetime.strptime(us_intraday_timestamp, '%H:%M:%S.%f').time()

            # Check if we need to sell 10 seconds before market close
            if time_obj >= market_close_time_for_scalp and scalp_trade_active and not (scalp_stop_loss_hit or scalp_profit_taker_hit):
                scalp_sell_price = price
                scalp_sell_time = us_intraday_timestamp
                logging.info(f"Scalp trade for {ticker}: Selling 10 seconds before market close at {scalp_sell_price}")
                break  # Exit loop after selling

            # Calculate price distance percentage
            price_distance_percentage = ((price - buy_price) / buy_price)

            # Check if buy price is reached for both main and scalp trades
            if not buy_price_reached and price >= (buy_price * 0.997) and price < (buy_price * 1.003) and (Industry_RSPercentile < 30.0 or Industry_RSPercentile > 60.0):

                # Check if we've reached the daily buy limit
                current_date = formatted_buy_date
                if current_date not in daily_buy_count:
                    daily_buy_count[current_date] = 0

                if daily_buy_count[current_date] < daily_buy_limit:
                    # Proceed with the buy
                    buy_price = price
                    pipeline.hset(key, 'BuyPrice', buy_price)
                    pipeline.hset(key, 'BuyDateTime', us_intraday_timestamp)
                    logging.info(f"Ticker: {ticker}, Price: {buy_price}, New Timestamp: {us_intraday_timestamp}")

                    if Daily_RSI == 'Y':
                        rsi_value_for_previous_day, rsi_date_used = get_rsi_value_for_previous_available_day(formatted_buy_date, rsi_values_by_date)
                        if rsi_value_for_previous_day is None:
                            logging.warning(f"No RSI data for QQQ before date {formatted_buy_date}")
                            return
                        if rsi_value_for_previous_day < 70:
                            daily_buy_count[current_date] += 1
                            logging.info(f"Buy count for {current_date}: {daily_buy_count[current_date]}")
                            buy_price_reached = True
                            logging.info(f"Ticker RSI Buy Price Reached!: {ticker}, RSI Value: {rsi_value_for_previous_day}, Price: {price}")
                            # Start scalp trade
                            scalp_trade_active = True
                            scalp_buy_price = buy_price
                        else:
                            logging.info(f"Skipping buy for {ticker} on {current_date} due to RSI {rsi_value_for_previous_day} >= 70")
                            continue  # Skip to next price point

                    else:
                        daily_buy_count[current_date] += 1
                        logging.info(f"Buy count for {current_date}: {daily_buy_count[current_date]}")
                        buy_price_reached = True
                        logging.info(f"Buy Price Reached!: {ticker}, US Timestamp: {us_intraday_timestamp}, Price: {price}")
                        # Start scalp trade
                        scalp_trade_active = True
                        scalp_buy_price = buy_price
                else:
                    logging.info(f"Daily buy limit reached for {current_date}. Skipping buy for {ticker}.")
                    continue

            # Scalp trade logic
            if scalp_trade_active and not scalp_stop_loss_hit and not scalp_profit_taker_hit:
                # Check for profit taker
                if price_distance_percentage >= Scalp_Profit_Taker:
                    scalp_sell_price = price
                    scalp_sell_time = us_intraday_timestamp
                    scalp_profit_taker_hit = True
                    logging.info(f"Scalp Profit Taker hit for {ticker} at {scalp_sell_time}, Price: {scalp_sell_price}")
                    break  # Exit loop after selling

                # Check for stop loss
                if price_distance_percentage <= -Scalp_Stop_Loss:
                    scalp_sell_price = price
                    scalp_sell_time = us_intraday_timestamp
                    scalp_stop_loss_hit = True
                    logging.info(f"Scalp Stop Loss hit for {ticker} at {scalp_sell_time}, Price: {scalp_sell_price}")
                    break  # Exit loop after selling

        if not buy_price_reached:
            logging.info(f"Buy Price Not Reached by End of Buy Date!: {formatted_buy_date}, {ticker}")
            # Set buy price to "Not reached"
            pipeline.hset(key, 'BuyPrice', 'Not reached')
            pipeline.execute()
            return

        # Calculate scalp result
        if scalp_trade_active and scalp_sell_price and scalp_sell_time:
            scalp_result_percentage = ((scalp_sell_price - scalp_buy_price) / scalp_buy_price)
            scalp_result_dollar = scalp_result_percentage * position_size
            logging.info(f"Scalp trade result for {ticker}: {scalp_result_dollar}$ ({scalp_result_percentage * 100}%)")
            # Store in Redis
            pipeline.hset(key, mapping={
                'Scalp_SellPrice': scalp_sell_price,
                'Scalp_SellTime': scalp_sell_time,
                'Scalp_Result_$': scalp_result_dollar
            })
        else:
            # If scalp trade was not executed or no sell price found
            logging.info(f"Scalp trade for {ticker} did not result in a sell. Setting Scalp_Result_$ to 0.")
            pipeline.hset(key, 'Scalp_Result_$', 0.0)

        # Now, process subsequent dates using 1-minute data
        # Organize 1-minute data
        intraday_results_1m = ticker_intraday_data_1m['results']
        intraday_results_by_date_1m = {}
        for result in intraday_results_1m:
            timestamp = datetime.datetime.fromtimestamp(result['t']/1000)
            date_str = timestamp.strftime('%Y-%m-%d')
            if date_str not in intraday_results_by_date_1m:
                intraday_results_by_date_1m[date_str] = []
            intraday_results_by_date_1m[date_str].append(result)

        # Start from the day after the buy date
        start_date = buy_date_dt + datetime.timedelta(days=1)
        while start_date <= end_date:
            formatted_date = start_date.strftime('%Y-%m-%d')

            # Get intraday data for the date
            date_intraday_data = intraday_results_by_date_1m.get(formatted_date)
            if not date_intraday_data:
                logging.warning(f"No intraday data for {ticker} on {formatted_date}")
                start_date += datetime.timedelta(days=1)
                continue

            prices = [float(result['c']) for result in date_intraday_data]
            price_timestamps = [result['t'] for result in date_intraday_data]

            filtered_data = [
                ((datetime.datetime.fromtimestamp(ts / 1000).astimezone(eastern).strftime('%H:%M:%S')), price)
                for ts, price in zip(price_timestamps, prices)
                if market_close_time > datetime.datetime.fromtimestamp(ts / 1000).astimezone(eastern).time() > market_open_time
            ]

            # Iterate over prices for the current day
            for us_intraday_timestamp, price in filtered_data:
                time_object = us_intraday_timestamp
                if price < stop_loss:
                    logging.info(f"Ticker: {ticker}, Price<Stop loss: {price}, New Timestamp: {us_intraday_timestamp}, Stop loss: {stop_loss}, Time:{time_object}")
                    stop_loss_hit = True

                    risk_reward_after_sl = ((buy_price - price) / buy_price) * 100

                    if is_risk_reward_2_reached and is_risk_reward_5_reached:
                        stopped_value = -(risk_reward_after_sl / 100) / 4
                    elif is_risk_reward_2_reached:
                        stopped_value = -(risk_reward_after_sl / 100) / 2
                    else:
                        stopped_value = -risk_reward_after_sl / 100

                    # Store stopped value in Redis using pipeline
                    stopped_value_dollar = stopped_value * position_size if position_size else None
                    logging.info(f"Stop Loss Hit!:{ticker}, SL: {stop_loss}, New_Timestamp: {us_intraday_timestamp}, Timestamp: {formatted_date}, Price: {price}, Stopped Value: {stopped_value}, Stopped Value in Dollar: {stopped_value_dollar}")
                    pipeline.hset(key, mapping={
                        'StoppedValue': stopped_value,
                        'StoppedValue_$': stopped_value_dollar,
                        'StoppedDate': formatted_date,
                        'StoppedTime': us_intraday_timestamp,
                        'Stoppedprice': price,
                        'Status': 'Stopped',
                        'RemainingResult': ''
                    })
                    pipeline.execute()
                    return

                price_distance_percentage = ((price - buy_price) / buy_price) * 100

                if price_distance_percentage > risk_reward * 1 and not is_risk_reward_2_reached:
                    logging.info(f"RiskReward x 2 Reached! Timestamp: {formatted_date}, New Timestamp: {us_intraday_timestamp}")
                    is_risk_reward_2_reached = True
                    # Update stop loss to buy price if Moving_Stop_Loss is 'Y'
                    if Moving_Stop_Loss == 'Y':
                        stop_loss = buy_price  # Update stop loss to buy price
                        pipeline.hset(key, 'StopLoss', stop_loss)
                    # Batch hset commands for RiskReward2
                    pipeline.hset(key, mapping={
                        'RiskReward2Date': formatted_date,
                        'RiskReward2Time': us_intraday_timestamp,
                        'RiskReward2Price': price,
                        'RiskReward2Value': (price_distance_percentage / 2) / 100,
                        'RiskReward2Value_$': ((price_distance_percentage / 2) / 100) * position_size
                    })

                if price_distance_percentage > risk_reward * 2 and not is_risk_reward_5_reached:
                    logging.info(f"RiskReward x 5 Reached! Timestamp: {formatted_date}, New Timestamp: {us_intraday_timestamp}")
                    is_risk_reward_5_reached = True
                    # Batch hset commands for RiskReward5
                    pipeline.hset(key, mapping={
                        'RiskReward5Date': formatted_date,
                        'RiskReward5Time': us_intraday_timestamp,
                        'RiskReward5Price': price,
                        'RiskReward5Value': (price_distance_percentage / 4) / 100,
                        'RiskReward5Value_$': ((price_distance_percentage / 4) / 100) * position_size
                    })

                if is_risk_reward_5_reached and Moving_Stop_Loss == 'Y' and formatted_date != previous_day:
                    try:
                        previous_day = formatted_date
                        # Use pre-fetched EMA data
                        ema_data_for_ticker = ema_data.get(ticker)
                        if ema_data_for_ticker and 'results' in ema_data_for_ticker and 'values' in ema_data_for_ticker['results']:
                            ema_values = ema_data_for_ticker['results']['values']
                            # Find EMA value for the current date
                            ema_value_for_date = next((item['value'] for item in ema_values if datetime.datetime.fromtimestamp(item['timestamp'] / 1000).strftime('%Y-%m-%d') == formatted_date), None)
                            if ema_value_for_date:
                                latest_ema8 = ema_value_for_date
                                pipeline.hset(key, mapping={
                                    'SL_EMA8': latest_ema8,
                                    'StopLoss': latest_ema8
                                })
                                stop_loss = latest_ema8
                                logging.info(f"Latest daily SL_EMA8 for {ticker}: {latest_ema8}")
                            else:
                                logging.warning(f"No EMA8 value for {ticker} on {formatted_date}")
                        else:
                            logging.warning(f"No EMA8 data for {ticker}")
                    except Exception as e:
                        logging.error(f"Error occurred while retrieving daily EMA8 for {ticker}: {e}")

            start_date += datetime.timedelta(days=1)

        # If stop loss was not hit, update "Still in" status
        if not stop_loss_hit:
            logging.info("Still in! Updating status...")
            pipeline.hset(key, 'Status', 'Still in')
            value_to_set = ((price_distance_percentage * 1) / 100)
            if is_risk_reward_2_reached and is_risk_reward_5_reached:
                value_to_set = ((price_distance_percentage * 1) / 4) / 100
            elif is_risk_reward_2_reached:
                value_to_set = ((price_distance_percentage * 1) / 2) / 100

            # Store Remaining result in Redis using pipeline
            value_to_set_dollar = value_to_set * position_size if position_size else None
            pipeline.hset(key, mapping={
                'RemainingResult': value_to_set,
                'RemainingResult_$': value_to_set_dollar
            })
        else:
            # If stop loss was hit, remove "Still in" status
            logging.info("Stop loss hit! Removing 'Still in' status...")
            pipeline.hset(key, mapping={
                'Status': 'Stopped',
                'RemainingResult': ''
            })

        # Execute all batched commands
        pipeline.execute()

    else:
        logging.error(f"Missing data for key: {key_decoded}")

def calculate_final_result():
    # Use a pipeline for batch processing
    pipeline = redis_db.pipeline()
    # Get all keys matching the pattern
    keys = redis_db.scan_iter(match=f"{node_name}:*")

    # Iterate over each key
    for key in keys:
        key_decoded = key.decode('utf-8')
        logging.info(f"Processing key: {key_decoded}")

        try:
            # Retrieve data from Redis
            fields = redis_db.hkeys(key)
        except redis.exceptions.ResponseError as e:
            logging.error(f"Redis ResponseError: {e}")
            continue

        # Skip the key 'backtesting_tool:columns'
        if b'backtesting_tool:columns' in fields:
            logging.info("Skipping key 'backtesting_tool:columns'")
            continue

        # Calculate FinalResult_$
        stopped_value_dollar = redis_db.hget(key, 'StoppedValue_$')
        risk_reward_2_value_dollar = redis_db.hget(key, 'RiskReward2Value_$')
        risk_reward_5_value_dollar = redis_db.hget(key, 'RiskReward5Value_$')
        remaining_result_dollar = redis_db.hget(key, 'RemainingResult_$')
        scalp_result_dollar = redis_db.hget(key, 'Scalp_Result_$')

        # Sum up available values
        final_result_dollar = 0.0
        if stopped_value_dollar:
            final_result_dollar += float(stopped_value_dollar.decode('utf-8'))
        if risk_reward_2_value_dollar:
            final_result_dollar += float(risk_reward_2_value_dollar.decode('utf-8'))
        if risk_reward_5_value_dollar:
            final_result_dollar += float(risk_reward_5_value_dollar.decode('utf-8'))
        if remaining_result_dollar:
            final_result_dollar += float(remaining_result_dollar.decode('utf-8'))
        # Do not include scalp_result_dollar here anymore
        # if scalp_result_dollar:
        #     final_result_dollar += float(scalp_result_dollar.decode('utf-8'))

        # Store FinalResult_$ in Redis using pipeline
        pipeline.hset(key, 'FinalResult_$', final_result_dollar)

    # Execute all batched commands
    pipeline.execute()

def limit_max_buys_per_day():
    # Dictionary to store tickers grouped by date
    tickers_by_date = {}

    # Get all keys matching the pattern
    keys = redis_db.scan_iter(match=f"{node_name}:*")

    # Group tickers by BuyDate and BuyDateTime
    for key in keys:
        key_decoded = key.decode('utf-8')
        buy_date = redis_db.hget(key, 'BuyDate')
        buy_date_time = redis_db.hget(key, 'BuyDateTime')

        if buy_date and buy_date_time:
            buy_date_str = buy_date.decode('utf-8')
            buy_time_str = buy_date_time.decode('utf-8')

            # Append ticker and BuyDateTime to the dictionary grouped by BuyDate
            if buy_date_str not in tickers_by_date:
                tickers_by_date[buy_date_str] = []

            tickers_by_date[buy_date_str].append((key, buy_time_str))

    # Use a pipeline for batch processing
    pipeline = redis_db.pipeline()

    # Process each date and sort tickers by BuyDateTime
    for buy_date_str, tickers_list in tickers_by_date.items():
        # Sort tickers by BuyDateTime (time) for the day
        tickers_list_sorted = sorted(tickers_list, key=lambda x: x[1])

        # Keep the first Max_Buys tickers for each day
        tickers_to_keep = tickers_list_sorted[:Max_Buys]
        tickers_to_cut = tickers_list_sorted[Max_Buys:]

        # Process tickers that exceeded Max_Buys limit
        for ticker_key, _ in tickers_to_cut:
            # Retrieve FinalResult_$ and move it to Above_Max_Buys
            final_result_dollar = redis_db.hget(ticker_key, 'FinalResult_$')

            if final_result_dollar:
                pipeline.hset(ticker_key, mapping={
                    'Above_Max_Buys': final_result_dollar,
                    'FinalResult_$': ''
                })
                logging.info(f"Ticker {ticker_key.decode('utf-8')} exceeded Max_Buys limit for {buy_date_str}. FinalResult_$ moved to Above_Max_Buys.")

    # Execute all batched commands

    pipeline.execute()

def sum_final_results_and_log():
    # Sum up all FinalResult_$ values
    total_final_result = 0.0
    total_scalp_result = 0.0
    keys = redis_db.scan_iter(match=f"{node_name}:*")

    for key in keys:
        final_result_dollar = redis_db.hget(key, 'FinalResult_$')
        scalp_result_dollar = redis_db.hget(key, 'Scalp_Result_$')
        if final_result_dollar:
            total_final_result += float(final_result_dollar.decode('utf-8'))
        if scalp_result_dollar:
            total_scalp_result += float(scalp_result_dollar.decode('utf-8'))

    # Get Max_Buys value
    max_buys_value = Max_Buys
    position_size = get_position_size()

    # Write to Final_Results.txt
    with open('Final_Results.txt', 'a') as f:
        f.write(f"Total FinalResult_$ (without scalp trades): {total_final_result}, Total Scalp_Result_$: {total_scalp_result}, Max_Buys: {max_buys_value}, Position_size: {position_size}, Scalp_Profit_Taker: {Scalp_Profit_Taker}, Scalp_Stop_Loss: {Scalp_Stop_Loss}, Market Open: {market_open_time}\n")
    logging.info(f"Total FinalResult_$: {total_final_result}, Total Scalp_Result_$: {total_scalp_result}, Max_Buys: {max_buys_value} logged to Final_Results.txt")

import datetime
import logging

def collect_and_sort_trade_events():
    keys = list(redis_db.scan_iter(match=f"{node_name}:*"))
    trade_events = []
    position_size = get_position_size()
    if position_size is None:
        logging.error("Position size is not set. Exiting calculation.")
        return [], 0, 0

    # Get Capital from Redis
    capital_value = redis_db.hget('Site_Configuration', 'Capital')
    if capital_value:
        capital = float(capital_value.decode('utf-8'))
    else:
        logging.error("Capital not found in Site_Configuration.")
        return [], 0, 0

    for key in keys:
        key_decoded = key.decode('utf-8')
        fields_raw = redis_db.hgetall(key)
        if not fields_raw:
            continue
        fields = {k.decode('utf-8'): v.decode('utf-8') for k, v in fields_raw.items()}

        ticker = fields.get('Ticker')
        buy_date = fields.get('BuyDate')
        buy_time = fields.get('BuyDateTime')
        buy_price = fields.get('BuyPrice')
        final_result = fields.get('FinalResult_$')

        # **Begin: Update FinalResult_$ based on Above_Max_Buys if necessary**
        if final_result in ('0', '0.0', ''):
            above_max_buys = fields.get('Above_Max_Buys')
            if above_max_buys and above_max_buys.strip() != '':
                final_result = above_max_buys
                logging.info(f"FinalResult_$ for key {key_decoded} set to Above_Max_Buys value: {final_result}")
            else:
                logging.debug(f"Above_Max_Buys not present or empty for key {key_decoded}. FinalResult_$ remains as: {final_result}")
        # **End: Update FinalResult_$**

        # Combine date and time into datetime objects
        events = []

        # Buy event
        if buy_date and buy_time and buy_price != "Not reached":
            buy_datetime_str = f"{buy_date} {buy_time}"
            try:
                # Adjust parsing based on your data format
                if '-' in buy_date:
                    buy_datetime = datetime.datetime.strptime(buy_datetime_str, '%Y-%m-%d %H:%M:%S.%f')
                else:
                    buy_datetime = datetime.datetime.strptime(buy_datetime_str, '%Y%m%d %H:%M:%S.%f')
            except ValueError:
                # Try without milliseconds
                if '-' in buy_date:
                    buy_datetime = datetime.datetime.strptime(buy_datetime_str, '%Y-%m-%d %H:%M:%S')
                else:
                    buy_datetime = datetime.datetime.strptime(buy_datetime_str, '%Y%m%d %H:%M:%S')
            events.append({
                'datetime': buy_datetime,
                'type': 'buy',
                'key': key_decoded,
                'ticker': ticker,
                'value': None,  # Omit Value for buys
                'details': {
                    'BuyPrice': buy_price,
                }
            })
        elif buy_price == "Not reached":
            # Treat as skipped buy with zeroed values
            events.append({
                'datetime': None,  # No datetime since buy didn't occur
                'type': 'buy_skipped',
                'key': key_decoded,
                'ticker': ticker,
                'value': 0.0,
                'details': {}
            })

        # RiskReward2 event
        rr2_date = fields.get('RiskReward2Date')
        rr2_time = fields.get('RiskReward2Time')
        rr2_value = fields.get('RiskReward2Value_$')
        if rr2_date and rr2_time and rr2_value:
            rr2_datetime_str = f"{rr2_date} {rr2_time}"
            try:
                rr2_datetime = datetime.datetime.strptime(rr2_datetime_str, '%Y-%m-%d %H:%M:%S')
            except ValueError:
                rr2_datetime = datetime.datetime.strptime(rr2_datetime_str, '%Y-%m-%d %H:%M:%S.%f')
            events.append({
                'datetime': rr2_datetime,
                'type': 'risk_reward_2',
                'key': key_decoded,
                'ticker': ticker,
                'value': float(rr2_value),
                'details': {}
            })

        # RiskReward5 event
        rr5_date = fields.get('RiskReward5Date')
        rr5_time = fields.get('RiskReward5Time')
        rr5_value = fields.get('RiskReward5Value_$')
        if rr5_date and rr5_time and rr5_value:
            rr5_datetime_str = f"{rr5_date} {rr5_time}"
            try:
                rr5_datetime = datetime.datetime.strptime(rr5_datetime_str, '%Y-%m-%d %H:%M:%S')
            except ValueError:
                rr5_datetime = datetime.datetime.strptime(rr5_datetime_str, '%Y-%m-%d %H:%M:%S.%f')
            events.append({
                'datetime': rr5_datetime,
                'type': 'risk_reward_5',
                'key': key_decoded,
                'ticker': ticker,
                'value': float(rr5_value),
                'details': {}
            })

        # Stopped event
        stopped_date = fields.get('StoppedDate')
        stopped_time = fields.get('StoppedTime')
        stopped_value = fields.get('StoppedValue_$')
        if stopped_date and stopped_time and stopped_value:
            stopped_datetime_str = f"{stopped_date} {stopped_time}"
            try:
                stopped_datetime = datetime.datetime.strptime(stopped_datetime_str, '%Y-%m-%d %H:%M:%S')
            except ValueError:
                stopped_datetime = datetime.datetime.strptime(stopped_datetime_str, '%Y-%m-%d %H:%M:%S.%f')
            # **Ensure final_result is a float, default to 0.0 if conversion fails**
            try:
                final_result_float = float(final_result)
            except (ValueError, TypeError):
                final_result_float = 0.0
                logging.warning(f"FinalResult_$ for key {key_decoded} is invalid. Defaulting to 0.0.")
            events.append({
                'datetime': stopped_datetime,
                'type': 'stopped',
                'key': key_decoded,
                'ticker': ticker,
                'value': float(stopped_value),
                'details': {
                    'FinalResult_$': final_result_float
                }
            })

        trade_events.extend(events)

    # Sort all events by datetime, skipping None datetimes (skipped buys)
    sorted_events = sorted(
        [event for event in trade_events if event['datetime'] is not None],
        key=lambda x: x['datetime']
    )

    return sorted_events, capital, position_size

def simulate_portfolio_management(sorted_events, capital, position_size, Max_Buys):
    current_capital = capital
    calculated_result = 0.0
    portfolio_trades = []

    # To track active positions (tickers we have bought but not yet fully sold)
    active_positions = set()
    skipped_buys = set()  # To keep track of keys for which buys were skipped before execution

    # Dictionary to track the number of buys per day
    buy_count_per_day = {}

    # Helper function to safely convert to float
    def safe_float_conversion(value, default=0.0):
        try:
            return float(value)
        except (TypeError, ValueError):
            return default

    for event in sorted_events:
        event_type = event['type']
        value = event['value']
        key = event['key']  # Ensure key is correctly formatted with colons
        ticker = event['ticker']
        event_datetime = event['datetime']
        datetime_str = event_datetime.strftime('%Y-%m-%d %H:%M:%S')
        details = event.get('details', {})

        # Log the key being processed
        logging.debug(f"Processing key: {key}")

        # Extract the date part for tracking buys per day
        event_date_str = event_datetime.strftime('%Y-%m-%d')

        # Initialize the buy count for the day if not already present
        if event_date_str not in buy_count_per_day:
            buy_count_per_day[event_date_str] = 0

        # Check if this key was previously marked as skipped before execution
        if (key, event_date_str) in skipped_buys:
            # If the buy was skipped before execution, set relevant values to zero
            action = 'Buy Skipped (Previously Skipped)'
            zeroed_values = {
                'StoppedValue_$': 0.0,
                'RiskReward2Value_$': 0.0,
                'RiskReward5Value_$': 0.0,
                'Capital_change': 0.0
            }
            trade_record = {
                'Key': key,
                'Datetime': datetime_str,
                'Action': action,
                'Ticker': ticker,
                'Value': 0.0,
                'Current Capital': current_capital,
                'Calculated_Result_$': calculated_result,
                'Change_to_capital': 0.0  # No change
            }
            trade_record.update(zeroed_values)
            portfolio_trades.append(trade_record)
            logging.info(f"Skipping event for {ticker} at {datetime_str} because the buy was previously skipped.")
            continue

        # Initialize Change_to_capital for the event
        change_to_capital = 0.0

        # Handle Buy Event
        if event_type == 'buy':
            # Check if the daily buy limit has been reached
            if buy_count_per_day[event_date_str] >= Max_Buys:
                # Skip the buy and mark the key as skipped before execution
                action = 'Buy Skipped (Max Buys Reached)'
                skipped_buys.add((key, event_date_str))
                logging.info(f"Skipping buy for {ticker} at {datetime_str} as Max_Buys ({Max_Buys}) reached for {event_date_str}.")

                # Ensure that subsequent values for this key are set to zero
                zeroed_values = {
                    'StoppedValue_$': 0.0,
                    'RiskReward2Value_$': 0.0,
                    'RiskReward5Value_$': 0.0,
                    'Capital_change': 0.0
                }
                trade_record = {
                    'Key': key,
                    'Datetime': datetime_str,
                    'Action': action,
                    'Ticker': ticker,
                    'Value': 0.0,
                    'Current Capital': current_capital,
                    'Calculated_Result_$': calculated_result,
                    'Change_to_capital': 0.0  # No change
                }
                trade_record.update(zeroed_values)
                portfolio_trades.append(trade_record)
                continue

            # Check if there's enough capital to execute the buy
            if current_capital >= position_size:
                # Execute the buy
                current_capital -= position_size
                change_to_capital = -position_size
                active_positions.add((key, event_date_str))
                action = 'Buy'
                logging.info(f"Buying {ticker} at {datetime_str}. Current Capital: {current_capital}")

                # Increment the buy count for the day
                buy_count_per_day[event_date_str] += 1
                logging.info(f"Buy count for {event_date_str}: {buy_count_per_day[event_date_str]}")
            else:
                # Skip buy due to insufficient capital and mark this key as skipped before execution
                action = 'Buy Skipped (Insufficient Capital)'
                skipped_buys.add((key, event_date_str))
                logging.info(f"Skipping buy for {ticker} at {datetime_str} due to insufficient capital.")

                # Ensure that subsequent values for this key are set to zero
                zeroed_values = {
                    'StoppedValue_$': 0.0,
                    'RiskReward2Value_$': 0.0,
                    'RiskReward5Value_$': 0.0,
                    'Capital_change': 0.0
                }
                trade_record = {
                    'Key': key,
                    'Datetime': datetime_str,
                    'Action': action,
                    'Ticker': ticker,
                    'Value': 0.0,
                    'Current Capital': current_capital,
                    'Calculated_Result_$': calculated_result,
                    'Change_to_capital': 0.0  # No change
                }
                trade_record.update(zeroed_values)
                portfolio_trades.append(trade_record)
                continue

            # Record the buy event
            trade_record = {
                'Key': key,
                'Datetime': datetime_str,
                'Action': action,
                'Ticker': ticker,
                'Value': 0.0,  # Omit Value by setting to 0.0
                'Current Capital': current_capital,
                'Calculated_Result_$': calculated_result,
                'Change_to_capital': change_to_capital
            }

            # Include additional details if available
            trade_record.update(details)
            portfolio_trades.append(trade_record)

        # Handle Stopped Event Logic
        elif event_type == 'stopped':
            # Safely retrieve FinalResult_$
            final_result_raw = value
            #details.get('FinalResult_$')
            final_result_value = safe_float_conversion(final_result_raw, default=0.0)

            # Retrieve other necessary fields
            rr2_date = redis_db.hget(key, 'RiskReward2Date')
            rr5_date = redis_db.hget(key, 'RiskReward5Date')

            # Determine the condition based on presence of RR2 and RR5
            rr2_present = bool(rr2_date and rr2_date.decode('utf-8').strip())
            rr5_present = bool(rr5_date and rr5_date.decode('utf-8').strip())

            # Determine Capital change based on conditions
            if not rr2_present and not rr5_present:
                if final_result_value < 0:
                    capital_change = position_size + final_result_value
                else:
                    capital_change = (position_size / 4) + final_result_value
            elif rr2_present and not rr5_present:
                capital_change = (position_size / 2) + final_result_value
            elif rr2_present and rr5_present:
                capital_change = (position_size / 4) + final_result_value
            else:
                capital_change = 0.0  # Default case

            # Adjust Current Capital
            current_capital += capital_change
            change_to_capital = capital_change
            calculated_result += final_result_value  # Add to Calculated_Result_$

            # Handle negative capital
            if current_capital < 0:
                logging.warning(f"Current Capital dropped below 0: {current_capital}. Skipping further buys until capital is positive.")
                # Implement logic to skip further buys until capital is positive
                # This can be handled by checking in the buy event section

            action = 'Stopped'
            logging.info(f"Processing stop for {ticker} at {datetime_str}. final_result_raw: {final_result_raw}, final_result_value: {final_result_value}, Capital Change: {capital_change}, Current Capital: {current_capital}, Calculated Result: {calculated_result}")

            # Record the stopped event
            trade_record = {
                'Key': key,
                'Datetime': datetime_str,
                'Action': action,
                'Ticker': ticker,
                'Value': value,  # StoppedValue_$
                'Current Capital': current_capital,
                'Calculated_Result_$': calculated_result,
                'Change_to_capital': change_to_capital,
                'FinalResult_$': final_result_value
            }

            # Include additional details if available
            trade_record.update(details)
            portfolio_trades.append(trade_record)

        # Handle RiskReward2 Event
        elif event_type == 'risk_reward_2':
            rr2_value = safe_float_conversion(value, default=0.0)
            capital_change = (position_size / 2) + rr2_value
            current_capital += capital_change
            change_to_capital = capital_change
            calculated_result += rr2_value
            action = 'RiskReward2'
            logging.info(f"Processing RiskReward2 for {ticker} at {datetime_str}. Capital Change: {capital_change}, Current Capital: {current_capital}, Calculated Result: {calculated_result}")

            # Record the risk_reward_2 event
            trade_record = {
                'Key': key,
                'Datetime': datetime_str,
                'Action': action,
                'Ticker': ticker,
                'Value': rr2_value,  # RiskReward2Value_$
                'Current Capital': current_capital,
                'Calculated_Result_$': calculated_result,
                'Change_to_capital': capital_change
            }

            # Include additional details if available
            trade_record.update(details)
            portfolio_trades.append(trade_record)

        # Handle RiskReward5 Event
        elif event_type == 'risk_reward_5':
            rr5_value = safe_float_conversion(value, default=0.0)
            capital_change = (position_size / 4) + rr5_value
            current_capital += capital_change
            change_to_capital = capital_change
            calculated_result += rr5_value
            action = 'RiskReward5'
            logging.info(f"Processing RiskReward5 for {ticker} at {datetime_str}. Capital Change: {capital_change}, Current Capital: {current_capital}, Calculated Result: {calculated_result}")
            # Record the risk_reward_5 event
            trade_record = {
                'Key': key,
                'Datetime': datetime_str,
                'Action': action,
                'Ticker': ticker,
                'Value': rr5_value,  # RiskReward5Value_$
                'Current Capital': current_capital,
                'Calculated_Result_$': calculated_result,
                'Change_to_capital': capital_change
            }

            # Include additional details if available
            trade_record.update(details)
            portfolio_trades.append(trade_record)

        # Handle Skipped Buy Event
        elif event_type == 'buy_skipped':
            action = 'Buy Skipped'
            zeroed_values = {
                'StoppedValue_$': 0.0,
                'RiskReward2Value_$': 0.0,
                'RiskReward5Value_$': 0.0,
                'Capital_change': 0.0
            }
            trade_record = {
                'Key': key,
                'Datetime': datetime_str,
                'Action': action,
                'Ticker': ticker,
                'Value': 0.0,
                'Current Capital': current_capital,
                'Calculated_Result_$': calculated_result,
                'Change_to_capital': 0.0  # No change
            }
            trade_record.update(zeroed_values)
            portfolio_trades.append(trade_record)
            logging.info(f"Buy was skipped for {ticker} at {datetime_str}. Zeroing relevant fields.")
            continue

        else:
            action = f"{event_type.capitalize()} Ignored"
            logging.info(f"Ignoring {event_type} for {ticker} at {datetime_str} as no active position.")
            # No change to capital
            change_to_capital = 0.0

            # Record the ignored event
            trade_record = {
                'Key': key,
                'Datetime': datetime_str,
                'Action': action,
                'Ticker': ticker,
                'Value': 0.0,  # Omit Value by setting to 0.0
                'Current Capital': current_capital,
                'Calculated_Result_$': calculated_result,
                'Change_to_capital': change_to_capital
            }

            # Include additional details if available
            trade_record.update(details)
            portfolio_trades.append(trade_record)

    return calculated_result, portfolio_trades

def write_portfolio_results(calculated_result, portfolio_trades):
    # Append Calculated_Result_$ to Final_Results.txt
    with open('Final_Results.txt', 'a') as f:
        f.write(f"Total Calculated_Result_$ (from main trades): {calculated_result}\n")

    # Write trades to Portfolio_Trades.txt
    with open('Portfolio_Trades.txt', 'w') as f:
        for trade in portfolio_trades:
            # Convert trade dict to a comma-separated string
            # Ensure that if 'Value' is None or 0.0, it's appropriately represented
            trade_line = ','.join(f"{k}:{v}" for k, v in trade.items())
            f.write(trade_line + '\n')

    logging.info(f"Calculated_Result_$: {calculated_result} logged to Final_Results.txt")
    logging.info("Portfolio trades written to Portfolio_Trades.txt")

def main():
    # Collect all keys
    keys = list(redis_db.scan_iter(match=f"{node_name}:*"))

    # Collect tickers and their buy dates
    ticker_date_ranges = {}
    rsi_dates = {'start_date': None, 'end_date': None}

    current_date = datetime.datetime.now()
    api_data_end_date = current_date - datetime.timedelta(days=1)  # Assuming data is available up to yesterday

    for key in keys:
        key_decoded = key.decode('utf-8')
        parts = key_decoded.split(":")
        if len(parts) != 3:
            logging.error(f"Invalid key format: {key_decoded}")
            continue

        ticker = parts[2]
        # Retrieve data from Redis
        if key_decoded in redis_fields_cache:
            fields = redis_fields_cache[key_decoded]
        else:
            fields_raw = redis_db.hgetall(key)
            if not fields_raw:
                logging.error(f"Key {key_decoded} has no fields.")
                continue
            fields = {k.decode('utf-8'): v.decode('utf-8') for k, v in fields_raw.items()}
            redis_fields_cache[key_decoded] = fields

        buy_date_str = fields.get('BuyDate')
        before_buy_date_str = fields.get('BeforeBuyDate')
        if not buy_date_str:
            logging.error(f"No BuyDate for key: {key_decoded}")
            continue

        # Adjust date parsing based on format
        if '-' in buy_date_str:
            buy_date = datetime.datetime.strptime(buy_date_str, '%Y-%m-%d')
        else:
            buy_date = datetime.datetime.strptime(buy_date_str, '%Y%m%d')


        # Check if buy_date is in the future
        if buy_date > api_data_end_date:
            logging.error(f"Buy date {buy_date} is in the future. Skipping key: {key_decoded}")
            continue

        end_date = api_data_end_date  # Use the capped end date

        # Collect buy dates per ticker
        if ticker not in ticker_date_ranges:
            ticker_date_ranges[ticker] = {'buy_dates': set(), 'end_date': end_date}
        ticker_date_ranges[ticker]['buy_dates'].add(buy_date)

        # Collect RSI dates
        if buy_date_str:
            buy_date_formatted = datetime.datetime.strptime(buy_date_str, '%Y%m%d') if '-' not in buy_date_str else buy_date
            if rsi_dates['start_date'] is None or buy_date_formatted < rsi_dates['start_date']:
                rsi_dates['start_date'] = buy_date_formatted
            if rsi_dates['end_date'] is None or buy_date_formatted > rsi_dates['end_date']:
                rsi_dates['end_date'] = buy_date_formatted

        if before_buy_date_str:
            try:
                if '-' in before_buy_date_str:
                    before_buy_date_formatted = datetime.datetime.strptime(before_buy_date_str, '%Y-%m-%d')
                else:
                    before_buy_date_formatted = datetime.datetime.strptime(before_buy_date_str, '%Y%m%d')
            except ValueError:
                logging.error(f"Invalid BeforeBuyDate format: {before_buy_date_str}")
                continue
            if rsi_dates['start_date'] is None or before_buy_date_formatted < rsi_dates['start_date']:
                rsi_dates['start_date'] = before_buy_date_formatted
            if rsi_dates['end_date'] is None or before_buy_date_formatted > rsi_dates['end_date']:
                rsi_dates['end_date'] = before_buy_date_formatted

    # Adjust rsi_dates if they are None
    if rsi_dates['start_date'] is None:
        rsi_dates['start_date'] = datetime.datetime.now() - datetime.timedelta(days=365)
    if rsi_dates['end_date'] is None:
        rsi_dates['end_date'] = datetime.datetime.now()

    # Build ticker_date_ranges with all buy dates
    for ticker in ticker_date_ranges:
        buy_dates = ticker_date_ranges[ticker]['buy_dates']
        min_buy_date = min(buy_dates)
        ticker_date_ranges[ticker]['start_date'] = min_buy_date

    # Now, fetch all the required data
    loop = asyncio.get_event_loop()
    intraday_data, rsi_data, ema_data = loop.run_until_complete(fetch_all_data(ticker_date_ranges, rsi_dates))

    # Now, process the keys in parallel
    calculate_result_with_data = partial(calculate_result_for_key, intraday_data=intraday_data, rsi_data=rsi_data, ema_data=ema_data)
    with ThreadPoolExecutor(max_workers=10) as executor:
        executor.map(calculate_result_with_data, keys)

    # Call the functions
    calculate_final_result()
    limit_max_buys_per_day()

    # New additions
    sum_final_results_and_log()
    # Collect and sort events
    sorted_events, capital, position_size = collect_and_sort_trade_events()
    # Check if capital and position_size are valid
    if capital == 0 or position_size == 0:
        logging.error("Capital or Position Size is zero. Exiting.")
        return
    # Simulate portfolio management
    calculated_result, portfolio_trades = simulate_portfolio_management(sorted_events, capital, position_size, Max_Buys)
    # Write results
    write_portfolio_results(calculated_result, portfolio_trades)

if __name__ == "__main__":
    main()