Modify and produce the complete script such that files to be downloaded and processed up to 202408 (YYYYMM) have the current format (PUBLIC_DVD_P5MIN_INTERCONNECTORSOLN_YYYYMM010000.zip).
Files to be downloaded from and including 202408 have the following format: PUBLIC_ARCHIVE#P5MIN_INTERCONNECTORSOLN#FILE01#YYYYMM010000.zip and the corresponding CSV file is in the format:
PUBLIC_ARCHIVE#P5MIN_INTERCONNECTORSOLN#FILE01#YYYYMM010000.CSV

There have been two changes that need to be adjusted in the code:
1) There are now 4 additional fields in the database table: local_price_adjustment_export, locally_constrained_export, local_price_adjustment_import, locally_constrained_import
2) The order of the fields in the csv file have changed. Here is the csv field order for files from and including 202408:
I,P5MIN,INTERCONNECTORSOLN,4,RUN_DATETIME,INTERCONNECTORID,INTERVAL_DATETIME,METEREDMWFLOW,MWFLOW,MWLOSSES,MARGINALVALUE,VIOLATIONDEGREE,MNSP,EXPORTLIMIT,IMPORTLIMIT,MARGINALLOSS,EXPORTGENCONID,IMPORTGENCONID,FCASEXPORTLIMIT,FCASIMPORTLIMIT,LASTCHANGED,LOCAL_PRICE_ADJUSTMENT_EXPORT,LOCALLY_CONSTRAINED_EXPORT,LOCAL_PRICE_ADJUSTMENT_IMPORT,LOCALLY_CONSTRAINED_IMPORT,INTERVENTION

As previously the first four fields are ignored. The "INTERVENTION" field is now the last field, but its location in the database has not changed.
These fields also need to be imported from the csv file: LOCAL_PRICE_ADJUSTMENT_EXPORT,LOCALLY_CONSTRAINED_EXPORT,LOCAL_PRICE_ADJUSTMENT_IMPORT,LOCALLY_CONSTRAINED_IMPORT

Produce the whole new script

#!/usr/bin/env python3
# File name structure: PUBLIC_DVD_P5MIN_INTERCONNECTORSOLN_YYYYMM010000.zip
# URL structure: https://nemweb.com.au/Data_Archive/Wholesale_Electricity/MMSDM/YYYY/MMSDM_YYYY_MM/MMSDM_Historical_Data_SQLLoader/DATA/

import os
import sys
from pathlib import Path
import requests
import zipfile
import csv
import psycopg2
from psycopg2 import sql, OperationalError
import configparser
from datetime import datetime
from tenacity import retry, stop_after_attempt, wait_fixed, before_sleep_log, retry_if_exception_type, RetryError
import logging
import argparse

# --- Global Constants ---
PROGRAM_VER = "dl_interconnect_5m_archive_1.0.3"
URL_BASE = "https://nemweb.com.au/Data_Archive/Wholesale_Electricity/MMSDM/"

# --- Paths ---
CWD = Path(__file__).resolve().parent
DATA_DIR = CWD / "temp" 
LOG_DIR = CWD / "log"  
LOGFILE = LOG_DIR / "dl_interconnect_5m_archive.log"  

HEADERS = {
    "User-Agent": "Mozilla/5.0",
    "Accept-Encoding": "gzip, deflate"
}

# --- Utilities ---
def ensure_dir_exists(path: Path):
    path.mkdir(parents=True, exist_ok=True)

ensure_dir_exists(DATA_DIR)
ensure_dir_exists(LOG_DIR)


def write_log(text: str, file: Path = LOGFILE):
    with open(file, 'a') as f:
        f.write(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} - {text}\n")


def print_and_log(text: str, err: Exception = None):
    print(text)
    write_log(text)
    if err:
        write_log(f"Error: {err}")


def print_psycopg2_exception(err: Exception):
    _, _, tb = sys.exc_info()
    line_num = tb.tb_lineno
    msg = f"psycopg2 ERROR: {err} on line {line_num}\npgerror: {err.pgerror}\npgcode: {err.pgcode}"
    print_and_log(msg, err)


def config(filename: str = 'aemo_config.ini', section: str = 'postgresql') -> dict:
    parser = configparser.ConfigParser()
    parser.read(filename)

    if parser.has_section(section):
        return dict(parser.items(section))
    
    raise Exception(f"Section {section} not found in {filename}")


def connect_db() -> psycopg2.extensions.connection | None:
    try:
        params = config()
        return psycopg2.connect(**params)
    except OperationalError as err:
        print_psycopg2_exception(err)
        return None


@retry(
    stop=stop_after_attempt(3), 
    wait=wait_fixed(2), 
    before_sleep=before_sleep_log(logging.getLogger(__name__), logging.WARNING),
    retry=retry_if_exception_type(requests.RequestException)
)
def download_file(url: str, dest_path: Path, filename: str, encoded_filename: str) -> Path | None:
    try:
        response = requests.get(url + encoded_filename, timeout=20, headers=HEADERS)
        response.raise_for_status()
        file_path = dest_path / filename
        file_path.write_bytes(response.content)
        print_and_log(f"Downloaded: {url + encoded_filename}")
        return file_path
    except requests.RequestException as e:
        print_and_log(f"Error downloading {url + encoded_filename}: {e}", e)
        raise


def unzip_file(zip_path: Path, extract_to: Path) -> Path | None:
    try:
        with zipfile.ZipFile(zip_path, 'r') as z:
            z.extractall(extract_to)
            csv_file = zip_path.stem + ".CSV"
            csv_path = extract_to / csv_file
            print_and_log(f"Unzipped: {csv_file}")
            return csv_path
    except zipfile.BadZipFile as e:
        print_and_log(f"Error unzipping {zip_path}: {e}", e)
    return None


def process_csv(csv_path: Path):
    conn = connect_db()
    if not conn:
        print_and_log("Database connection failed, exiting process_csv")
        return

    try:
        with conn.cursor() as cur:
            with open(csv_path, 'r') as file:
                csv_reader = csv.reader(file)
                headers = next(csv_reader)  # Skip header
                print_and_log(f"Processing CSV file: {csv_path.name}")

                rows = []
                row_count = 0

                for row in csv_reader:
                    row_count += 1
                    if not row or row[0] != "D" or len(row) < 22:
                        print_and_log(f"Skipping invalid row {row_count}: {row}")
                        continue

                    try:
                        run_datetime = datetime.strptime(row[4].strip().strip('"'), "%Y/%m/%d %H:%M:%S") if row[4].strip() else None
                        intervention = int(row[5].strip()) if row[5].strip() else None
                        interconnectorid = row[6].strip()
                        interval_datetime = datetime.strptime(row[7].strip().strip('"'), "%Y/%m/%d %H:%M:%S") if row[7].strip() else None
                        meteredmwflow = float(row[8].strip()) if row[8].strip() else None
                        mwflow = float(row[9].strip()) if row[9].strip() else None
                        mwlosses = float(row[10].strip()) if row[10].strip() else None
                        marginalvalue = float(row[11].strip()) if row[11].strip() else None
                        violationdegree = float(row[12].strip()) if row[12].strip() else None
                        mnsp = int(row[13].strip()) if row[13].strip() else None
                        exportlimit = float(row[14].strip()) if row[14].strip() else None
                        importlimit = float(row[15].strip()) if row[15].strip() else None
                        marginalloss = float(row[16].strip()) if row[16].strip() else None
                        exportgenconid = row[17].strip() if row[17].strip() else None
                        importgenconid = row[18].strip() if row[18].strip() else None
                        fcasexportlimit = float(row[19].strip()) if row[19].strip() else None
                        fcasimportlimit = float(row[20].strip()) if row[20].strip() else None
                        lastchanged = datetime.strptime(row[21].strip().strip('"'), "%Y/%m/%d %H:%M:%S") if row[21].strip() else None

                        rows.append((
                            run_datetime, intervention, interconnectorid, interval_datetime,
                            meteredmwflow, mwflow, mwlosses, marginalvalue, violationdegree, mnsp,
                            exportlimit, importlimit, marginalloss, exportgenconid, importgenconid,
                            fcasexportlimit, fcasimportlimit, lastchanged
                        ))

                    except Exception as e:
                        print_and_log(f"Error parsing row {row_count}: {row}. Error: {e}", e)
                        continue
                
                print_and_log(f"Parsed {len(rows)} valid rows from {csv_path.name}")

                if rows:
                    try:
                        cur.executemany(
                            """
                            INSERT INTO nem_interconnect_5m (
                                run_datetime, intervention, interconnectorid, interval_datetime,
                                meteredmwflow, mwflow, mwlosses, marginalvalue, violationdegree, mnsp,
                                exportlimit, importlimit, marginalloss, exportgenconid, importgenconid,
                                fcasexportlimit, fcasimportlimit, lastchanged
                            )
                            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                            ON CONFLICT (interconnectorid, interval_datetime) DO NOTHING
                            """,
                            rows
                        )
                        conn.commit()
                        print_and_log(f"{cur.rowcount} records inserted/updated from {csv_path.name}")
                    except Exception as e:
                        conn.rollback()
                        print_and_log(f"Error during executemany insert for {csv_path}: {e}", e)

                        # Detailed logging of problem rows (optional, for debugging only!)
                        for i, row_data in enumerate(rows):
                            try:
                                cur.execute(
                                    """
                                    INSERT INTO nem_interconnect_5m (
                                        run_datetime, intervention, interconnectorid, interval_datetime,
                                        meteredmwflow, mwflow, mwlosses, marginalvalue, violationdegree, mnsp,
                                        exportlimit, importlimit, marginalloss, exportgenconid, importgenconid,
                                        fcasexportlimit, fcasimportlimit, lastchanged
                                    )
                                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                                    ON CONFLICT (interconnectorid, interval_datetime) DO NOTHING
                                    """,
                                    row_data
                                )
                            except Exception as insert_err:
                                print_and_log(f"Detailed error on row {i+1}: {row_data}. Insert error: {insert_err}", insert_err)
                        conn.commit()  # Commit successfully inserted rows (if any)
                    finally:
                         pass # remove

    except Exception as e:
        conn.rollback()
        print_and_log(f"Outer error processing {csv_path}: {e}", e)
    finally:
        if conn:
            conn.close()
            print_and_log("Database connection closed.")
        
# --- Main Execution ---
def main(start_year_month: str, end_year_month: str):
    print_and_log(f"Program version: {PROGRAM_VER}")

    start_year, start_month = int(start_year_month[:4]), int(start_year_month[4:])
    end_year, end_month = int(end_year_month[:4]), int(end_year_month[4:])

    for year in range(start_year, end_year + 1):
        # Determine the month range for the given year
        if year == start_year and year == end_year:
            months = range(start_month, end_month + 1)
        elif year == start_year:
            months = range(start_month, 13)
        elif year == end_year:
            months = range(1, end_month + 1)
        else:
            months = range(1, 13)

        for month in months:
            month_str = f"{month:02d}"
            url = f"{URL_BASE}{year}/MMSDM_{year}_{month_str}/MMSDM_Historical_Data_SQLLoader/DATA/"
            # PUBLIC_DVD_P5MIN_INTERCONNECTORSOLN_YYYYMM010000.zip
            raw_filename = f"PUBLIC_DVD_P5MIN_INTERCONNECTORSOLN_{year}{month_str}010000.zip"
            encoded_filename = raw_filename.replace("#", "%23")
            zip_path = DATA_DIR / raw_filename

            # Skip if file already exists
            if zip_path.exists():
                print_and_log(f"{zip_path} already exists. Skipping download.")
            else:    
                try:
                    zip_path = download_file(url, DATA_DIR, raw_filename, encoded_filename)
                except RetryError as e:
                    print_and_log(f"Failed to download {url + encoded_filename} after 3 attempts: {e}", e)
                    continue
            
            if zip_path:
                csv_path = unzip_file(zip_path, DATA_DIR)
                if csv_path:
                    process_csv(csv_path)
                    zip_path.unlink(missing_ok=True)
                    csv_path.unlink(missing_ok=True)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Download and process AEMO rooftop PV data.")
    parser.add_argument("start_year_month", type=str, help="Start year and month in YYYYMM format")
    parser.add_argument("end_year_month", type=str, help="End year and month in YYYYMM format")
    
    args = parser.parse_args()
    main(args.start_year_month, args.end_year_month)