setup influx instance 1.8, create dummy data, then create stale series and run the below cleanup script....the script may have some issues, so ensure you correct them as well....in the end i want a cleanup script that i can run on influx 1.8 to clean stale series import argparse
import json
import requests
import logging
import time
import os
import hashlib
import glob  # Add import for managing log files
import shutil  # Add import for disk usage monitoring
import threading  # Add import for threading
from urllib3.exceptions import InsecureRequestWarning
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from concurrent.futures import ThreadPoolExecutor, as_completed

# Suppress only the single InsecureRequestWarning from urllib3 needed for this script
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)

def parse_args():
    parser = argparse.ArgumentParser(description='Drop series from InfluxDB.')
    parser.add_argument('--host', required=True, help='Database host')
    parser.add_argument('--port', required=True, help='Database port')
    parser.add_argument('--user', required=True, help='Database user')
    parser.add_argument('--password', required=True, help='Database password')
    parser.add_argument('--database', required=True, help='Database name')
    parser.add_argument('--json_file', required=True, help='Path to JSON file')
    parser.add_argument('--measurement', required=True, help='Measurement name to drop series from')
    parser.add_argument('--limit', type=int, help='Number of keys to process from JSON file')
    parser.add_argument('--range', help='Range of keys to process (e.g., 100-200)')
    parser.add_argument('--ignore_cache', action='store_true', help='Ignore cache and process all keys from start')
    parser.add_argument('--safe', nargs='?', const=90, type=int, help='Safe CPU and memory usage percentage threshold (default: 90)')
    parser.add_argument('--debug', action='store_true', help='Enable debug prints')  # Add debug argument
    return parser.parse_args()

def debug_print(debug, message):
    """Print debug messages if debug mode is enabled."""
    if debug:
        print(message)

def setup_logger():
    log_dir = '.'  # Directory to store log files
    log_prefix = 'influx_series_drop'
    log_files = sorted(glob.glob(f"{log_dir}/{log_prefix}_*.log"))

    # Ensure only 5 log files are kept
    if len(log_files) >= 5:
        os.remove(log_files[0])  # Delete the oldest log file

    # Create a new log file with a timestamp
    log_filename = f"{log_dir}/{log_prefix}_{time.strftime('%Y%m%d_%H%M%S')}.log"
    logger = logging.getLogger('InfluxDBLogger')
    logger.setLevel(logging.WARNING)  # Set to WARNING for file logging

    # Remove any existing handlers to avoid duplicate logs
    if logger.hasHandlers():
        logger.handlers.clear()

    # FileHandler for ERROR and WARNING logs
    fh = logging.FileHandler(log_filename)
    fh.setLevel(logging.WARNING)
    file_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    fh.setFormatter(file_formatter)
    logger.addHandler(fh)

    return logger

def setup_session():
    session = requests.Session()
    retries = Retry(total=5, backoff_factor=1, status_forcelist=[502, 503, 504])
    session.mount('https://', HTTPAdapter(max_retries=retries))
    return session

def get_temp_file(json_file):
    """Generate a unique temporary file name based on the JSON file."""
    temp_dir = '.temp'
    os.makedirs(temp_dir, exist_ok=True)  # Ensure the temp directory exists
    temp_file = os.path.join(temp_dir, f'temp_processed_indices_{os.path.basename(json_file)}')
    return temp_file

def load_temp_file(temp_file):
    """Load indices from the temporary file if it exists."""
    if os.path.exists(temp_file):
        with open(temp_file, 'r') as f:
            return set(json.load(f))  # Load indices as a set
    return set()

def save_to_temp_file(temp_file, index):
    """Append a processed index to the temporary file."""
    with open(temp_file, 'a') as f:
        f.write(f"{index}\n")  # Write the index to the temp file

def update_cache_from_temp(temp_file, cache_file):
    """Update the cache using indices from the temporary file and delete the temp file."""
    if os.path.exists(temp_file):
        print(f"Processing temp file: {temp_file}")  # Debug print
        with open(temp_file, 'r') as f:
            temp_indices = {int(line.strip()) for line in f.readlines()}  # Read indices from temp file
        if os.path.exists(cache_file):
            with open(cache_file, 'r') as f:
                cache_indices = set(json.load(f))  # Load existing cache
        else:
            cache_indices = set()
        cache_indices.update(temp_indices)  # Update cache with temp indices
        with open(cache_file, 'w') as f:
            json.dump(list(cache_indices), f)  # Save updated cache
        os.remove(temp_file)  # Delete the temp file
        print(f"Temp file {temp_file} processed and deleted.")  # Debug print

def drop_series(host, port, user, password, protocol, database, measurement, where_clause, insecure, logger, index, key, keys_to_process, query_count, lock, temp_file, debug):
    try:
        debug_print(debug, f"drop_series called for index: {index}, key: {key}")  # Debug print
        if insecure:
            debug_print(debug, "Disabling SSL warnings.")  # Debug print
            requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

        url = f"{protocol}://{host}:{port}/query"
        query = f"DROP SERIES FROM {measurement} WHERE {where_clause}"
        debug_print(debug, f"Constructed query: {query}")  # Debug print

        params = {
            'q': query,
            'db': database,
            'u': user,
            'p': password
        }
        query = query.replace("\\", "\\\\")
        params['q'] = query
        debug_print(debug, f"Query parameters: {params}")  # Debug print

        start_time = time.time()
        try:
            debug_print(debug, f"Sending POST request to {url}")  # Debug print
            response = requests.post(url, params=params, verify=not insecure)
        except requests.exceptions.RequestException as e:
            logger.error(f"Request failed for index: {index}, key: {key}. Error: {e}")
            debug_print(debug, f"Request failed for index: {index}, key: {key}. Error: {e}")  # Debug print
            return 0  # Return 0 query time in case of failure

        end_time = time.time()
        query_time = end_time - start_time
        debug_print(debug, f"Query executed in {query_time:.2f} seconds.")  # Debug print

        with lock:
            debug_print(debug, f"Acquired lock for index: {index}")  # Debug print
            query_count[0] += 1
            save_to_temp_file(temp_file, index)
            debug_print(debug, f"Released lock for index: {index}")  # Debug print

        if response.status_code == 200:
            result = response.json()
            debug_print(debug, f"Response received: {result}")  # Debug print
            if 'results' in result and 'error' not in result['results'][0]:
                debug_print(debug, f"Series dropped successfully for index: {index}")  # Debug print
                print(f"Series dropped successfully. Time taken: {query_time:.2f} seconds. Deleted index: {index}. Remaining indices to process: {len(keys_to_process) - query_count[0]}")
            else:
                error_msg = result['results'][0].get('error', 'Unknown error')
                logger.error(f"Failed to drop series for key: {key}, query: {query}, where_clause: {where_clause}. Error: {error_msg}")
                debug_print(debug, f"Failed to drop series for index: {index}. Error: {error_msg}")  # Debug print
        else:
            logger.error(f"Failed to drop series for key: {key}, query: {query}, where_clause: {where_clause}. Response: {response.text}")
            debug_print(debug, f"Failed to drop series for index: {index}. Response: {response.text}")  # Debug print
        return query_time
    except Exception as e:
        logger.error(f"Unhandled exception in drop_series for index: {index}, key: {key}. Error: {e}")
        debug_print(debug, f"Unhandled exception in drop_series for index: {index}, key: {key}. Error: {e}")  # Debug print
        return 0  # Return 0 query time in case of failure

def get_cache_file(json_file):
    """Generate a unique cache file name based on the inode number of the JSON file."""
    cache_dir = '.cache'
    os.makedirs(cache_dir, exist_ok=True)  # Ensure the cache directory exists
    try:
        inode = os.stat(json_file).st_ino  # Get inode number of the file
    except AttributeError:
        # Fallback for systems without inode support
        inode = hashlib.md5(json_file.encode()).hexdigest()
    cache_file = os.path.join(cache_dir, f'cache_processed_indices_{inode}')  # Remove leading dot
    logging.debug(f"Cache file path: {cache_file}")  # Debug log for cache file path
    return cache_file

def load_cache(json_file):
    cache_file = get_cache_file(json_file)
    if os.path.exists(cache_file):
        with open(cache_file, 'r') as f:
            return set(json.load(f))  # Load indices as a set
    return set()

def save_cache(json_file, processed_indices, temp_file=None):
    """Save processed indices to the cache file and optionally delete the temp file."""
    cache_file = get_cache_file(json_file)
    try:
        with open(cache_file, 'w') as f:
            json.dump(list(processed_indices), f)  # Save indices as a list
        logging.info(f"Cache saved successfully to {cache_file}")  # Log cache save success

        # Delete the temp file if it exists
        if temp_file and os.path.exists(temp_file):
            print(f"Deleting temp file: {temp_file}")  # Debug print
            os.remove(temp_file)
            logging.info(f"Temporary file {temp_file} deleted after cache update.")
    except Exception as e:
        logging.error(f"Failed to save cache: {e}")  # Log any errors during cache save

def wait_for_safe_conditions(safe_threshold):
    """Wait until CPU and memory usage are below the safe threshold."""
    while True:
        # Get CPU usage using os.getloadavg (1-minute average load)
        cpu_usage = os.getloadavg()[0] / os.cpu_count() * 100
        # Get memory usage using shutil.disk_usage
        total_memory, used_memory, _ = shutil.disk_usage("/")
        memory_usage = (used_memory / total_memory) * 100

        if cpu_usage < safe_threshold and memory_usage < safe_threshold:
            break
        print(f"System stats - CPU usage: {cpu_usage:.2f}%, Memory usage: {memory_usage:.2f}%. Waiting for resources to be below {safe_threshold}%...")
        time.sleep(2)  # Wait for 2 seconds before rechecking

def escape_single_quotes(value):
    """Escape single quotes, backslashes, and special characters in a string for use in InfluxDB queries."""
    value = value.replace("\\", "\\\\")  # Escape backslashes
    value = value.replace("\n", "\\n")  # Escape newlines
    value = value.replace("=", "\\=")  # Escape equal signs
    value = value.replace("'", "''")  # Escape single quotes
    return value

def check_influx_connection(host, port, user, password, database, protocol, insecure, debug):
    """Check if InfluxDB is accessible and credentials are valid."""
    url = f"{protocol}://{host}:{port}/query"
    params = {
        'q': 'SHOW DATABASES',
        'u': user,
        'p': password
    }
    try:
        response = requests.get(url, params=params, verify=not insecure)
        if response.status_code == 200:
            databases = response.json().get('results', [{}])[0].get('series', [{}])[0].get('values', [])
            if any(database in db for db in databases):
                debug_print(debug, f"Successfully connected to InfluxDB. Database '{database}' is accessible.")
                return True
            else:
                print(f"Database '{database}' not found in InfluxDB.")
        else:
            print(f"Failed to connect to InfluxDB. Status code: {response.status_code}, Response: {response.text}")
    except Exception as e:
        print(f"Error connecting to InfluxDB: {e}")
    return False

def debug_active_threads():
    """Print all active threads for debugging."""
    print("Active threads:")
    for thread in threading.enumerate():
        print(f"Thread Name: {thread.name}, Alive: {thread.is_alive()}")

def main():
    args = parse_args()
    logger = setup_logger()  # Initialize the logger
    print("Script started.")  # Debug print

    debug_active_threads()  # Debug active threads at the start

    # Check InfluxDB connection
    if not check_influx_connection(args.host, args.port, args.user, args.password, args.database, 'https', True, args.debug):
        print("Unable to connect to InfluxDB. Exiting.")
        return

    measurement = args.measurement
    try:
        with open(args.json_file, 'r') as f:
            series = json.load(f)
        print(f"Loaded JSON file: {args.json_file}")
    except (IOError, json.JSONDecodeError) as e:
        logger.error(f"Failed to read JSON file: {e}")
        return

    cache_file = get_cache_file(args.json_file)
    temp_file = get_temp_file(args.json_file)

    # Update cache from temp file if it exists
    update_cache_from_temp(temp_file, cache_file)

    processed_indices = set()
    if not args.ignore_cache:
        processed_indices = load_cache(args.json_file)
        print(f"Loaded cache with {len(processed_indices)} processed indices.")

    keys = list(series.keys())
    total_keys = len(keys)
    print(f"Total keys in JSON: {total_keys}")

    # Determine the range of indices to process
    if args.range:
        try:
            start, end = map(int, args.range.split('-'))
            indices_to_check = range(start, end)
            print(f"Processing keys in range: {start}-{end}")
        except ValueError:
            logger.error("Invalid range format. Use start-end (e.g., 100-200).")
            return
    elif args.limit:
        # Find the next unprocessed range of size `--limit`
        start = 0
        while start in processed_indices:
            start += 1
        end = min(start + args.limit, total_keys)
        indices_to_check = range(start, end)
        print(f"Processing next {args.limit} keys starting from index {start}.")
    else:
        start, end = 0, total_keys
        indices_to_check = range(total_keys)
        print("Processing all keys.")

    # Exclude processed indices
    keys_to_process = [i for i in indices_to_check if i not in processed_indices]
    print(f"Found {len(keys_to_process)} keys to process after excluding cache.")
    print(f"Keys to process: {len(keys_to_process)}")  # Debug print

    total_time = 0
    query_count = [0]  # Use a list to allow updates within threads
    lock = threading.Lock()  # Create a lock for synchronizing updates
    loop_start_time = time.time()

    with ThreadPoolExecutor(max_workers=5) as executor:
        #debug_print(args.debug, "ThreadPoolExecutor initialized.")  # Debug print
        futures = []
        for index in keys_to_process:
            key = keys[index]
            parts = key.split(',')
            where_clause = ' AND '.join([
                f'"{kv[0]}"=\'{escape_single_quotes(kv[1])}\''  # Escape field names with double quotes and values with single quotes
                for part in parts[1:]
                for kv in [part.split('=', 1)] if len(kv) == 2
            ])
            try:
                future = executor.submit(
                    drop_series, args.host, args.port, args.user, args.password, 'https',
                    args.database, measurement, where_clause, True, logger, index, key, keys_to_process, query_count, lock, temp_file, args.debug  # Pass debug flag
                )
                futures.append(future)
                debug_print(args.debug, f"Task submitted for index: {index}, key: {key}")  # Debug print
            except Exception as e:
                logger.error(f"Error submitting task for index: {index}, key: {key}. Error: {e}")
                print(f"Error submitting task for index: {index}, key: {key}. Error: {e}")

        debug_print(args.debug, f"Starting Drop")  # Debug print
        debug_active_threads()
        futures_to_indices = {future: index for future, index in zip(futures, keys_to_process)}

        for future in as_completed(futures):
            index = futures_to_indices[future]  # Get the corresponding index for the completed future
            debug_print(args.debug, f"Processing completed for index: {index}")  # Debug print
            try:
                query_time = future.result()
                total_time += query_time
                processed_indices.add(index)  # Add index to processed indices only after success
            except Exception as e:
                key = keys[index]
                parts = key.split(',')
                where_clause = ' AND '.join([
                    f"{kv[0]}='{kv[1]}'" for part in parts[1:]
                    for kv in [part.split('=', 1)] if len(kv) == 2
                ])
                logger.error(f"Error during query execution for key: {key}, where_clause: {where_clause}. Error: {e}")
                print(f"Error during query execution for key: {key}, where_clause: {where_clause}. Error: {e}")

    print( "All tasks completed.")  # Debug print
    loop_end_time = time.time()
    save_cache(args.json_file, processed_indices, temp_file)  # Pass temp_file to delete it after cache update

    if query_count[0] > 0:
        average_time = (loop_end_time - loop_start_time) / query_count[0]
        print(f"Total time taken to drop all series: {loop_end_time - loop_start_time:.2f} seconds")
        print(f"Average time per query: {average_time:.2f} seconds")
        print(f"Total series dropped: {query_count[0]}")
        print(f"Processed indices range: {start} to {end - 1}")

        # Log the summary to the log file as INFO
        logger.info(f"Total time taken to drop all series: {loop_end_time - loop_start_time:.2f} seconds")
        logger.info(f"Average time per query: {average_time:.2f} seconds")
        logger.info(f"Total series dropped: {query_count[0]}")
        logger.info(f"Processed indices range: {start} to {end - 1}")

    print("Script finished.")  # Debug print
    logger.info("Script finished.")

    debug_active_threads()  # Debug active threads at the end

if __name__ == '__main__':
    main()