Modify this script so that there is the option to insert a range of dates (startdatetime, enddatetime) in the command line in the format YYYYMMDDHHmm:

#!/usr/bin/env python3
# Previously known as data_download_vX.X.X.py
"""
AEMO DATA DOWNLOADER – Dispatch SCADA pipeline (FULL DOWNLOAD, LIMITED, OVERWRITE)
-------------------------------------------------
• Scrapes http://nemweb.com.au/reports/CURRENT/Dispatch_SCADA/
• Downloads the X most recent *.zip files (specified via command line), sorted by timestamp
• Extracts the CSV inside each zip
• Inserts/Updates rows into nem_dispatch (ON CONFLICT DO UPDATE)
• Cleans up temp files and logs every step

Prerequisites
-------------
1. Python 3.9+ with requests, beautifulsoup4, psycopg2-binary
2. Command line argument specifying the number of files to download.

"""
import os
import sys
import zipfile
import time
import requests
import configparser
import psycopg2
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from psycopg2 import OperationalError
import argparse  # Import the argparse module
import re  # Import the regular expression module

# ── Constants ──────────────────────────────────────────────
PROGRAM_VER = "dl_dispatch_v3.3.4"  # Modified version name
URL_DISPATCH = "http://nemweb.com.au/reports/CURRENT/Dispatch_SCADA/"
HEADERS = {"User-Agent": "Mozilla/5.0"}
TIMESTAMP = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

# ── Paths ─────────────────────────────────────────────────
CWD = os.path.abspath(os.path.dirname(__file__)) + "/"
DATA_DIR = os.path.join(CWD, "temp")
LOG_DIR = os.path.join(CWD, "log")  # Log directory
LOGFILE = os.path.join(LOG_DIR, "dl_dispatch.log")
INI_PATH = os.path.join(CWD, "aemo_config.ini")

os.makedirs(DATA_DIR, exist_ok=True)

# ── Logging helper ────────────────────────────────────────
def log(msg: str):
    print(msg)
    with open(LOGFILE, "a", encoding="utf-8") as fh:
        fh.write(f"{datetime.now():%Y-%m-%d %H:%M:%S}  {msg}\n")

# ── DB helpers ────────────────────────────────────────────

def connect_db():
    cfg = configparser.ConfigParser()
    if not cfg.read(INI_PATH):
        log(f"ERROR: Unable to read {INI_PATH}")
        return None
    if not cfg.has_section("postgresql"):
        log(f"ERROR: Missing [postgresql] section in {INI_PATH}")
        return None
    try:
        params = {k: v for k, v in cfg.items("postgresql")}
        return psycopg2.connect(**params)
    except (OperationalError, psycopg2.Error) as err:
        log(f"DB connection error: {err}")
        return None

def exec_sql(sql, fetch="none", ctx=""):
    conn = connect_db()
    if not conn:
        return [] if fetch != "none" else False
    try:
        with conn:
            with conn.cursor() as cur:
                cur.execute(sql)
                if fetch == "all":
                    return cur.fetchall()
                if fetch == "one":
                    return cur.fetchone()
        return True
    except psycopg2.Error as err:
        log(f"psycopg2 error in {ctx}: {err}")
        return [] if fetch != "none" else False
    finally:
        conn.close()

# ── Web scraping ──────────────────────────────────────────

def fetch_scada_filenames():
    try:
        r = requests.get(URL_DISPATCH, headers=HEADERS, timeout=30)
        r.raise_for_status()
        soup = BeautifulSoup(r.text, "html.parser")
        files = [a.text.strip() for a in soup.find_all('a')
                 if a.text.endswith('.zip') and 'PUBLIC_DISPATCHSCADA_' in a.text]
        log(f"Found {len(files)} SCADA zip files online.")
        return files
    except Exception as e:
        log(f"ERROR fetching SCADA page: {e}")
        return []

# ── Download / unzip / import ─────────────────────────────

def download_file(filename):
    url = URL_DISPATCH + filename
    local_path = os.path.join(DATA_DIR, filename)
    if os.path.exists(local_path):
        return local_path  # already downloaded
    try:
        r = requests.get(url, headers=HEADERS, stream=True, timeout=60)
        r.raise_for_status()
        with open(local_path, "wb") as fh:
            for chunk in r.iter_content(8192):
                fh.write(chunk)
        log(f"Downloaded {filename}")
        return local_path
    except Exception as e:
        log(f"Download failed for {filename}: {e}")
        return None

def unzip_get_csv(zip_path):
    try:
        with zipfile.ZipFile(zip_path) as z:
            csv_name = z.namelist()[-1]
            z.extract(csv_name, DATA_DIR)
            return os.path.join(DATA_DIR, csv_name)
    except Exception as e:
        log(f"Unzip failed for {zip_path}: {e}")
        return None

def import_csv_dispatch(csv_path):
    conn = connect_db()
    if not conn:
        return False
    try:
        inserted = 0
        with open(csv_path, 'r') as fh, conn:
            with conn.cursor() as cur:
                for line in fh:
                    if line.startswith(('C', 'I')):
                        continue
                    parts = line[24:].replace('/', '-').strip().split(',')
                    if len(parts) < 4:
                        continue
                    try:
                        # Ensure parts[3] is a valid datetime string for lastchanged
                        lastchanged = parts[3]
                        cur.execute(
                            """
                            INSERT INTO nem_dispatch (settlement_date, duid, scada_value, lastchanged)
                            VALUES (%s,%s,%s,%s)
                            ON CONFLICT (settlement_date, duid) DO UPDATE
                            SET scada_value = EXCLUDED.scada_value, lastchanged = EXCLUDED.lastchanged
                            """,
                            (parts[0], parts[1], parts[2], lastchanged))  # Pass lastchanged
                        inserted += cur.rowcount
                    except Exception as e:
                        conn.rollback()
                        log(f"SQL Error: {e}")  # Log the specific SQL error
                        raise  # Re-raise the exception after logging
        log(f"Imported/Updated {inserted} rows from {os.path.basename(csv_path)}")
        return True
    except Exception as e:
        log(f"Import error {csv_path}: {e}")
        return False
    finally:
        conn.close()

# ── Helper function to extract datetime from filename ─────────────────────────────
def extract_datetime_from_filename(filename):
    """
    Extracts datetime object from the filename.
    Filenames are like PUBLIC_DISPATCHSCADA_202505261805_0000000464981457.zip
    """
    #match = re.search(r"PUBLIC_DISPATCHSCADA_(\d{12})_(\d{12})\.zip", filename) # Original regex
    match = re.search(r"PUBLIC_DISPATCHSCADA_(\d{12})_.*\.zip", filename)  # Corrected regex
    if match:
        start_time_str = match.group(1)  # e.g., 202505261805
        try:
            start_time = datetime.strptime(start_time_str, "%Y%m%d%H%M")
            return start_time
        except ValueError:
            log(f"Could not parse datetime from filename: {filename}")
            return None
    else:
        log(f"Could not extract datetime pattern from filename: {filename}")
        return None

# ── Main flow ─────────────────────────────────────────────
if __name__ == "__main__":
    # Set up argument parser
    parser = argparse.ArgumentParser(description="Download and import AEMO Dispatch SCADA data.")
    parser.add_argument("-x", "--num_files", type=int, default=0,
                        help="Number of most recent files to download (0 for all)")

    args = parser.parse_args()
    num_files_to_download = args.num_files

    log(f"\nSTART {PROGRAM_VER}  cwd={CWD}")

    online_files = fetch_scada_filenames()

    # Sort files by datetime extracted from filename (newest first)
    online_files_sorted = sorted(online_files,
                                 key=extract_datetime_from_filename,
                                 reverse=True)  # Sort newest first

    if num_files_to_download > 0:
        to_get = online_files_sorted[:num_files_to_download]  # Get the 'x' most recent
        log(f"Downloading the {num_files_to_download} most recent files.")
    else:
        to_get = online_files_sorted  # Download all files
        log("Downloading all available files.")

    log(f"Files to download: {len(to_get)}")

    for zip_name in to_get:
        zip_path = download_file(zip_name)
        if not zip_path:
            continue
        csv_path = unzip_get_csv(zip_path)
        if csv_path and import_csv_dispatch(csv_path):
            os.remove(zip_path)
            os.remove(csv_path)
            log(f"Cleaned {zip_name}")

    log("FINISHED")