Analyze this code and remove all redundant logic and code.
CODE:
from psycopg2 import OperationalError
import re
import threading
import time
import datetime
import psycopg2
import requests
from bs4 import BeautifulSoup
from psycopg2 import sql
from tqdm import tqdm
import multiprocessing
import json

def check_case_exists(cnx, case_name, scc_id):
    print(f'checking {case_name} {scc_id}')
    while True:
        try:
            cursor = cnx.cursor()
            # select_query = (
            #     "SELECT COUNT(*) FROM scc_table WHERE case_name = %s AND scc_id = %s"
            # )

            # # Execute the SELECT query with the provided scc_id
            # cursor.execute(select_query, (case_name, scc_id))

            select_query = (
                "SELECT COUNT(*) FROM scc_table WHERE scc_id = %s"
            )

            # Execute the SELECT query with the provided scc_id
            cursor.execute(select_query, scc_id)

            # Fetch the result
            result = cursor.fetchone()

            # Check if any rows exist with the provided scc_id
            if result[0] > 0:
                return True  # Case exists
            else:
                return False  # Case does not exist

        except psycopg2.Error as e:
            error_code = e.pgcode
            if error_code == '25P02':  # InFailedSqlTransaction error code
                # Wait for a short while before retrying
                time.sleep(0.5)
            else:
                #print("Error while checking case existence in PostgreSQL:", e)
                return False  # Return False if an error occurs


def connect_with_retry():
    while True:
        try:
            conn = psycopg2.connect(
       user="postgres",
        host="127.0.0.1",
        port=5433,
        database="postgres",
        password= "EHyg5DpRJ8sd8CtJ2hXsa8RZmPTXRyfJ621",
            )
            return conn
        except OperationalError as e:
            print(e)
            time.sleep(0.5)


def add_entry(
    cnx,
    bench_name,
    court_name,
    case_name,
    case_no,
    date,
    advocates,
    citations,
    case_text,
    scc_id,
):
    while True:
        try:
            if check_case_exists(cnx, case_name, scc_id):
                return
            cursor = cnx.cursor()
            insert_query = """
               INSERT INTO scc_table (bench_name, court_name, case_name, case_no, date, advocates, citations, case_text, scc_id)
                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
            """
            data = (
                bench_name,
                court_name,
                case_name,
                case_no,
                date,
                advocates,
                citations,
                case_text,
                scc_id,
            )
            cursor.execute(insert_query, data)

            cnx.commit()
            cursor.close()
            break  # Break the loop if the operation is successful
        except psycopg2.Error as e:
            error_code = e.pgcode
            if error_code == '25P02':  # InFailedSqlTransaction error code
                # Wait for a short while before retrying
                time.sleep(0.5)
            else:
                #print("Error while inserting data into PostgreSQL scc_table:", e)
                cnx.rollback()
                break  # Break the loop on other errors



# Create the scc_table
def create_scc_table(conn):
    cur = conn.cursor()
    cur.execute("""
        CREATE TABLE IF NOT EXISTS scc_table (
            id SERIAL PRIMARY KEY,
            bench_name TEXT,
            court_name TEXT,
            case_name TEXT,
            case_no TEXT,
            date DATE,
            advocates TEXT[],
            citations TEXT[],
            case_text TEXT,
            scc_id TEXT UNIQUE
        )
    """)
    conn.commit()
    cur.close()


def create_citations_table(conn):
    cur = conn.cursor()
    cur.execute("""
        CREATE TABLE IF NOT EXISTS citations (
            id SERIAL PRIMARY KEY,
            title TEXT,
            text TEXT,
            unique_id TEXT UNIQUE,
            type TEXT
        )
    """)
    conn.commit()
    cur.close()


def create_scc_citations_table(conn):
    cur = conn.cursor()
    cur.execute("""
        CREATE TABLE IF NOT EXISTS scc_citations (
            scc_id SERIAL,
            citation_id SERIAL,
            FOREIGN KEY (scc_id) REFERENCES scc_table (id) ON DELETE CASCADE,
            FOREIGN KEY (citation_id) REFERENCES citations (id) ON DELETE CASCADE
        )
    """)
    conn.commit()
    cur.close()


def create_scraped_table():
    # Establish a connection to the PostgreSQL database
    connection = connect_with_retry()
    cursor = connection.cursor()

    # Create the scraped table if it doesn't exist
    cursor.execute("""
    CREATE TABLE IF NOT EXISTS scraped (
        id SERIAL PRIMARY KEY,
        court_type VARCHAR(255) NOT NULL,
        court_name VARCHAR(255) NOT NULL,
        year INTEGER NOT NULL,
        month INTEGER NOT NULL,
        day INTEGER NOT NULL,
        completed BOOLEAN DEFAULT FALSE
    )
    """)

    # Commit the transaction and close the connection
    connection.commit()
    connection.close()


def add_to_scraped(conn,court_type, court_name, year, month, day, completed=False):
    while True:
        try:
            # Establish a connection to the PostgreSQL database
            cursor = conn.cursor()

            # Insert a new row into the scraped table
            cursor.execute("""
            INSERT INTO scraped (court_type, court_name, year, month, day, completed)
            VALUES (%s, %s, %s, %s, %s, %s)
            """, (court_type, court_name, year, month, day, completed))

            # Commit the transaction and close the connection
            conn.commit()
            cursor.close()
            break
        except:
            pass

# Insert a new citation into the citations table
def insert_citation(conn, title, text, unique_id, type):
    while True:
        try:
            cur = conn.cursor()
            cur.execute(
                sql.SQL(
                    "INSERT INTO citations (title, text, unique_id, type) VALUES (%s, %s, %s, %s)"
                ),
                (title, text, unique_id, type),
            )
            conn.commit()
            cur.close()
            break  # Break the loop if the operation is successful
        except psycopg2.Error as e:
            error_code = e.pgcode
            if error_code == '25P02':  # InFailedSqlTransaction error code
                # Wait for a short while before retrying
                time.sleep(0.5)
            else:
                #print("Error while inserting data into PostgreSQL citation:", e)
                conn.rollback()
                break  # Break the loop on other errors



def check_unique_id_exists(conn, unique_id):
    cur = conn.cursor()
    cur.execute(
        "SELECT COUNT(*) FROM citations WHERE unique_id = %s", (unique_id,))
    count = cur.fetchone()[0]
    cur.close()
    return count > 0


# Associate a citation with a case in the scc_citations table
def associate_citation(conn, scc_id, unique_id):
    while True:
        try:
            cur = conn.cursor()

            # Find the id for scc_id in scc_table
            cur.execute(
                "SELECT id FROM scc_table WHERE scc_id = %s",
                (scc_id,)
            )
            scc_table_id = cur.fetchone()[0] if cur.rowcount > 0 else None

            # Find the id for unique_id in citations
            cur.execute(
                "SELECT id FROM citations WHERE unique_id = %s",
                (unique_id,)
            )
            citations_id = cur.fetchone()[0] if cur.rowcount > 0 else None

            # Insert into scc_citations table if both scc_id and unique_id are found
            if scc_table_id is not None and citations_id is not None:
                cur.execute(
                    "INSERT INTO scc_citations (scc_id, citation_id) VALUES (%s, %s)",
                    (scc_table_id, citations_id),
                )
                conn.commit()

            cur.close()
            break  # Break the loop if the operation is successful
        except psycopg2.Error as e:
            error_code = e.pgcode
            if error_code == '25P02':  # InFailedSqlTransaction error code
                # Wait for a short while before retrying
                time.sleep(0.5)
            else:
                #print("Error while associating citation:", e)
                conn.rollback()
                break  # Break the loop on other errors


def update_completed_status(conn,court_type, court_name, year, month, day, completed=True):
    while True:
        try:
            cursor = conn.cursor()

            # Update the completed status of the row
            cursor.execute("""
            UPDATE scraped
            SET completed = %s
            WHERE court_type = %s AND court_name = %s AND year = %s AND month = %s AND day = %s
            """, (completed, court_type, court_name, year, month, day))

            # Commit the transaction and close the connection
            cursor.commit()
            conn.close()
        except psycopg2.Error as e:
            error_code = e.pgcode
            if error_code == '25P02':  # InFailedSqlTransaction error code
                # Wait for a short while before retrying
                time.sleep(0.5)
            else:
                # print("Error while inserting data into PostgreSQL citation:", e)
                conn.rollback()
                break  # Break the loop on other errors


def get_last_completed_day(court_type, court_name, year):
    while True:
        try:
            # Establish a connection to the PostgreSQL database
            connection = connect_with_retry()
            cursor = connection.cursor()

            # Query the database for the last completed day
            cursor.execute("""
            SELECT month, day
            FROM scraped
            WHERE court_type = %s AND court_name = %s AND year = %s AND completed = TRUE
            ORDER BY year DESC, month DESC, day DESC
            LIMIT 1
            """, (court_type, court_name, year))

            # Fetch the result
            result = cursor.fetchone()

            # Close the connection
            connection.close()

            # Return the last completed day
            return result if result else None
        except:
            pass

def check_login():
    check_url = "https://www.scconline.com/Members/SearchResult.aspx"
    if requests.get(check_url).text.find("SessionExpired") == -1:
        return True
    else:
        return False


def login_to_website(url, username, password):
    # Define the payload for the login request
    payload = {"loginId": username, "pass": password, "force": True}

    # Perform a GET request to obtain the x-access-token cookie
    response = requests.get(url)
    x_access_token = response.cookies.get("x-access-token")

    # Parse the response content using BeautifulSoup
    soup = BeautifulSoup(response.content, "html.parser")
    # Extract the 'crisp' value from the script tags
    list_containers = soup.find_all("script", type="text/javascript")
    crisp = list_containers[1].text.split("'")[3]

    # Set the cookie and headers for the POST request
    cookie = {"x-access-token": x_access_token}
    headers = {
        "X-Access-Token": ".".join(x_access_token.split(".")[:-1]),
        "Vid": "league",
    }

    # Perform the POST request to login
    response = requests.post(
        url + "/home/login", data=payload, cookies=cookie, headers=headers
    )

    # Return the response
    return response, crisp


def get_aspxauth_cookie(url, enc):
    headers = {
        "Host": "www.scconline.com",
        "Content-Length": "0",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.118 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "Sec-Fetch-Site": "same-site",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-User": "?1",
        "Sec-Fetch-Dest": "document",
        "Sec-Ch-Ua": '"Not-A.Brand";v="99", "Chromium";v="124"',
        "Sec-Ch-Ua-Mobile": "?0",
        "Sec-Ch-Ua-Platform": '"Windows"',
        "Referer": "https://www.scconline.com/",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9",
        "Priority": "u=0, i",
    }

    response = requests.get(
        url + "/ApplicationLogin.aspx?enc=" + enc, headers=headers)

    # print(response.request.headers)
    ASPXAUTH = response.request.headers["Cookie"] #.split(";")[3].split("=")[1]

    return ASPXAUTH


def get_all_courts(ASPXAUTH, type_court):
    url = "https://www.scconline.com/Searcher.svc/SearchBrowseTree"
    headers = {
        "Host": "www.scconline.com",
        "Cookie": ASPXAUTH,
        "Sec-Ch-Ua": '"Chromium";v="123", "Not:A-Brand";v="8"',
        "Sec-Ch-Ua-Mobile": "?0",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.122 Safari/537.36",
        "Content-Type": "application/json; charset=UTF-8",
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "X-Requested-With": "XMLHttpRequest",
        "Request-Id": "|b9+GB.eAlTH",
        "Sec-Ch-Ua-Platform": '"Windows"',
        "Origin": "https://www.scconline.com",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Dest": "empty",
        "Referer": "https://www.scconline.com/Members/BrowseResult.aspx",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9",
        "Priority": "u=4, i",
    }
    data = {
        "searchDetails": {
            "QueryText": f'Node2:"{type_court}"',
            "ReturnOnExit": False,
            "RequiredRows": 500,
            "SelectedCourt": "",
            "HighlightTree": True,
            "IsIclrContent": True,
            "IsJudiciaryPackage": False,
            "SearchText": "",
            "IsMootCourtAccessible": "false",
            "CountryName": "india",
            "SearchType": "Judgments (Court Wise)",
            "IsBrowseBySearch": False,
            "SearchField": "Node2",
            "UserSubscribedAddonList": ["NoAddOn"],
            "QueryType": " India",
            "parentNode": "Node1",
            "HasChildren": True,
        }
    }  

    # print('headers',headers)
    # print('body',data)

    while True:
        try:
            response = requests.post(url, headers=headers, json=data)
            print(response.status_code)
            print(response.text)
            return response.json()
        except:
            time.sleep(0.5)
            pass

def get_subcourts(ASPXAUTH,type_court,court):
    url = "https://www.scconline.com/Searcher.svc/SearchBrowseTree"
    headers = {
        "Host": "www.scconline.com",
        "Cookie": ASPXAUTH,
        "Sec-Ch-Ua": '"Chromium";v="123", "Not:A-Brand";v="8"',
        "Sec-Ch-Ua-Mobile": "?0",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.122 Safari/537.36",
        "Content-Type": "application/json; charset=UTF-8",
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "X-Requested-With": "XMLHttpRequest",
        "Request-Id": "|b9+GB.eAlTH",
        "Sec-Ch-Ua-Platform": '"Windows"',
        "Origin": "https://www.scconline.com",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Dest": "empty",
        "Referer": "https://www.scconline.com/Members/BrowseResult.aspx",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9",
        "Priority": "u=4, i",
    }
    data = {
        "searchDetails": {
            "QueryText": f"Node3:\"{court}\" AND Node2:\"{type_court}\"",
            "ReturnOnExit": False,
            "RequiredRows": 500,
            "SelectedCourt": "0000001111111111100001100100000000110000011111111000111111000001000100101000000000000000000000000000011101111111111111110110011101111111001100000000001111110100110111111111111111000111111111011111111110001111101110011111001011011111",
            "HighlightTree": True,
            "IsIclrContent": True,
            "IsJudiciaryPackage": False,
            "SearchText": "",
            "IsMootCourtAccessible": "false",
            "CountryName": "india",
            "SearchType": "Judgments (Court Wise)",
            "IsBrowseBySearch": False,
            "SearchField": "Node3",
            "UserSubscribedAddonList": ["NoAddOn"],
            "QueryType": f"{court}",
            "parentNode": "Node2",
            "HasChildren": True
        }
    }

    while True:
        try:
            response = requests.post(url, headers=headers, json=data)
            # print(response.json())
            return response.text
        except:
            time.sleep(0.5)
            pass


def get_year_range_for_subcourt(aspxauth, court_name, subcourt_name,type_court):
    url = "https://www.scconline.com/Searcher.svc/SearchBrowseTree"
    headers = {
        "Host": "www.scconline.com",
        "Cookie": aspxauth, #f".ASPXAUTH={aspxauth}",
        "Content-Length": "728",
        "Sec-Ch-Ua": '"Not-A.Brand";v="99", "Chromium";v="124"',
        "Sec-Ch-Ua-Mobile": "?0",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.118 Safari/537.36",
        "Content-Type": "application/json; charset=UTF-8",
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "X-Requested-With": "XMLHttpRequest",
        "Request-Id": "|zioJ.zSW5y",
        "Sec-Ch-Ua-Platform": '"Windows"',
        "Origin": "https://www.scconline.com",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Dest": "empty",
        "Referer": "https://www.scconline.com/Members/BrowseResult.aspx",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9",
        "Priority": "u=4, i"
    }

    data = {
        'searchDetails': {
            "QueryText": f"Node4:\"{subcourt_name}\" AND Node3:\"{court_name}\" AND Node2:\"{type_court}\"",
            "ReturnOnExit": False,
            "RequiredRows": 500,
            "SelectedCourt": "0000001111111111100001100100000000110000011111111000111111000001000100101000000000000000000000000000011101111111111111110110011101111111001100000000001111110100110111111111111111000111111111011111111110001111101110011111001011011111",
            "HighlightTree": True,
            "IsIclrContent": True,
            "IsJudiciaryPackage": False,
            "SearchText": "",
            "IsMootCourtAccessible": "false",
            "CountryName": "india",
            "SearchType": "Judgments (Court Wise)",
            "IsBrowseBySearch": False,
            "SearchField": "Node4",
            "UserSubscribedAddonList": ["NoAddOn"],
            "QueryType": f"{subcourt_name}",
            "parentNode": "Node3",
            "HasChildren": True}
    }

    while True:
        try:
            response = requests.post(url, headers=headers, json=data)
            return response.json()
        except:
            time.sleep(0.5)
            pass


def get_all_case_titles_1(ASPXAUTH, type, court, day, month, year):
    url = "https://www.scconline.com/Searcher.svc/SearchBrowseTree"
    headers = {
        "Host": "www.scconline.com",
        "Cookie": ASPXAUTH,
        "Sec-Ch-Ua": '"Chromium";v="123", "Not:A-Brand";v="8"',
        "Sec-Ch-Ua-Mobile": "?0",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.122 Safari/537.36",
        "Content-Type": "application/json; charset=UTF-8",
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "X-Requested-With": "XMLHttpRequest",
        "Request-Id": "|HMpn+.KCek4",
        "Sec-Ch-Ua-Platform": '"Windows"',
        "Origin": "https://www.scconline.com",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Dest": "empty",
        "Referer": "https://www.scconline.com/Members/BrowseResult.aspx",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9",
        "Priority": "u=1, i",
    }
    data = {
        "searchDetails": {
            "RequiredRows": 10000,
            "SelectedCourt": "",
            "HighlightTree": True,
            "IsIclrContent": True,
            "CountryName": "india",
            "IsBrowseBySearch": False,
            "SearchField": "Date",
            "QueryType": f"{day}",
            "HasChildren": True,
            "SearchType": "Judgments (Court Wise)",
            "ReturnOnExit": False,
            "SelectedDocRelativePath": "",
            "parentNode": "Month",
            "Title": "",
            "IsMootCourtAccessible": "true",
            "SearchText": "",
            "QueryText": f'Date:"{day}" AND Month:"{month}" AND Year:"{year}" AND Node3:"{court}" AND Node2:"{type}"',
            "UserSubscribedAddonList": ["NoAddOn"],
        }
    }

    while True:
        try:
            response = requests.post(url, headers=headers, json=data)
            return response.json()
        except:
            time.sleep(0.5)
            pass

def get_all_case_titles_2(ASPXAUTH, type, court, subcourt,day, month, year):
    url = "https://www.scconline.com/Searcher.svc/SearchBrowseTree"
    headers = {
        "Host": "www.scconline.com",
        "Cookie": ASPXAUTH,
        "Sec-Ch-Ua": '"Chromium";v="123", "Not:A-Brand";v="8"',
        "Sec-Ch-Ua-Mobile": "?0",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.122 Safari/537.36",
        "Content-Type": "application/json; charset=UTF-8",
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "X-Requested-With": "XMLHttpRequest",
        "Request-Id": "|HMpn+.KCek4",
        "Sec-Ch-Ua-Platform": '"Windows"',
        "Origin": "https://www.scconline.com",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Dest": "empty",
        "Referer": "https://www.scconline.com/Members/BrowseResult.aspx",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9",
        "Priority": "u=1, i",
    }
    data = {
        "searchDetails": {
            "RequiredRows": 10000,
            "SelectedCourt": "",
            "HighlightTree": True,
            "IsIclrContent": True,
            "CountryName": "india",
            "IsBrowseBySearch": False,
            "SearchField": "Date",
            "QueryType": f"{day}",
            "HasChildren": True,
            "SearchType": "Judgments (Court Wise)",
            "ReturnOnExit": False,
            "SelectedDocRelativePath": "",
            "parentNode": "Month",
            "Title": "",
            "IsMootCourtAccessible": "true",
            "SearchText": "",
            "QueryText": f'Date:"{day}" AND Month:"{month}" AND Year:"{year}" AND Node4:"{subcourt}" Node3:"{court}" AND Node2:"{type}"',
            "UserSubscribedAddonList": ["NoAddOn"],
        }
    }


    while True:
        try:
            response = requests.post(url, headers=headers, json=data)
            return response.json()
        except:
            time.sleep(0.5)
            pass

def get_xml_path(ASPXAUTH, title):
    url = "https://www.scconline.com/Searcher.svc/SearchRelativePath"
    headers = {
        "Host": "www.scconline.com",
        "Cookie": ASPXAUTH,
        "Sec-Ch-Ua": '"Chromium";v="123", "Not:A-Brand";v="8"',
        "Sec-Ch-Ua-Mobile": "?0",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.122 Safari/537.36",
        "Content-Type": "application/json; charset=UTF-8",
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "X-Requested-With": "XMLHttpRequest",
        "Request-Id": "|HMpn+.KCek4",
        "Sec-Ch-Ua-Platform": '"Windows"',
        "Origin": "https://www.scconline.com",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Dest": "empty",
        "Referer": "https://www.scconline.com/Members/BrowseResult.aspx",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9",
        "Priority": "u=1, i",
    }

    data = {
        "searchDetails": {
            "QueryText": f'Title:"{title}"',
            "ReturnOnExit": False,
            "RequiredRows": 500,
            "SelectedCourt": "",
            "HighlightTree": True,
            "IsIclrContent": True,
            "IsJudiciaryPackage": False,
            "SearchText": "",
            "IsMootCourtAccessible": "false",
            "CountryName": "india",
            "SearchType": "Judgments (Court Wise)",
            "IsBrowseBySearch": False,
            "SearchField": "",
            "UserSubscribedAddonList": ["NoAddOn"],
            "QueryType": "",
            "parentNode": "Month",
            "HasChildren": False,
        }
    }


    while True:
        try:
            response = requests.post(url, headers=headers, json=data)
            return response.json()
        except:
            time.sleep(0.5)
            pass


def get_middle_panel(ASPXAUTH, xmlpath):
    url = "https://www.scconline.com/Searcher.svc/GetMiddlePanel"
    headers = {
        "Host": "www.scconline.com",
        "Cookie": ASPXAUTH,
        "Sec-Ch-Ua": "\"Chromium\";v=\"123\", \"Not:A-Brand\";v=\"8\"",
        "Sec-Ch-Ua-Mobile": "?0",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.122 Safari/537.36",
        "Content-Type": "application/json; charset=UTF-8",
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "X-Requested-With": "XMLHttpRequest",
        "Request-Id": "|HMpn+.KCek4",
        "Sec-Ch-Ua-Platform": "\"Windows\"",
        "Origin": "https://www.scconline.com",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Dest": "empty",
        "Referer": "https://www.scconline.com/Members/BrowseResult.aspx",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9",
        "Priority": "u=1, i"
    }
    xmlpath = xmlpath.replace("\\", "\\\\")
    data = {
        "searchDetails": {
            "packageGroupId": "1",
            "QueryText": f"\"{xmlpath}\"",
            "RequiredRows": 10,
            "SearchField": "RelativePath",
            "BuildTree": True,
            "CurrentPage": 1,
            "IsIclrContent": True,
            "SortType": "asc",
            "SearchFeature": "gSearch",
            "HasJumpTo": False,
            "IsMaintainHistory": "0",
            "SearchType": "Browse",
            "SearchWithSynStem": False,
            "QueryType": "browse",
            "SearchInResultQuery": [],
            "SelectedCourts": [],
            "FacetFilters": [],
            "SelectedCourt": "0000001111111111100001100100000000110000011111111000111111000001000100101000000000000000000000000000011101111111111111110110011101111111001100000000001111110100110111111111111111000111111111011111111110001111101110011111001011011111",
            "NearSlop": 10,
            "IsMootCourtAccessible": "false"
        }
    }


    while True:
        try:
            response = requests.post(url, headers=headers, json=data)
            return response.json()
        except:
            time.sleep(0.5)
            pass


def get_page_data(ASPXAUTH, xmlpath):
    url = "https://www.scconline.com/HelperServices/ServicesForCourtFunctionality.asmx/GetPageData"
    headers = {
        "Host": "www.scconline.com",
        "Cookie": ASPXAUTH,
        "Sec-Ch-Ua": '"Chromium";v="123", "Not:A-Brand";v="8"',
        "Sec-Ch-Ua-Mobile": "?0",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.122 Safari/537.36",
        "Content-Type": "application/json; charset=UTF-8",
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "X-Requested-With": "XMLHttpRequest",
        "Request-Id": "|HMpn+.KCek4",
        "Sec-Ch-Ua-Platform": '"Windows"',
        "Origin": "https://www.scconline.com",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Dest": "empty",
        "Referer": "https://www.scconline.com/Members/BrowseResult.aspx",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9",
        "Priority": "u=1, i",
    }

    data = {
        "searchDetails": {
            "QueryText": f'"{xmlpath}"',
            "SearchField": "RelativePath",
            "SelectedCourt": "",
            "QueryType": "browse",
            "SearchInResultQuery": [],
            "SearchFeature": "gSearch",
            "UserName": "ca2a23b9cbbcaa08009d889dfc725559",
        },
        "path": f"{xmlpath}",
        "subheadingOrCitation": "",
        "isHighlight": False,
        "valueForXslt": "",
        "sectionSearchingForXslt": "",
        "DisplayNameFromTree": "",
    }

    while True:
        try:
            response = requests.post(url, headers=headers, json=data)
            return response.json()
        except:
            time.sleep(0.5)
            pass


def get_data_citation(ASPXAUTH, type, id):
    url = "https://www.scconline.com/Searcher.svc/SearchForCitaView"
    headers = {
        "Cookie": ASPXAUTH,
        "Content-Type": "application/json; charset=UTF-8",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Dest": "empty",
        "Referer": "https://www.scconline.com/Members/NoteView.aspx?enc=SlRYVC0wMDAyODk3ODYxJiYmJiY0MCYmJiYmQnJvd3NlUGFnZQ==",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.118 Safari/537.36",
        "Origin": "https://www.scconline.com",
        "X-Requested-With": "XMLHttpRequest",
        "Sec-Ch-Ua": '"Not-A.Brand";v="99", "Chromium";v="124"',
        "Sec-Ch-Ua-Mobile": "?0",
        "Sec-Ch-Ua-Platform": '"Windows"',
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9",
        "Priority": "u=1, i",
    }
    if type:
        data = {
            "searchDetails": {
                "SearchField": f"{type}",
                "QueryText": f"{id}",
                "RequiredRows": 50,
                "BuildTree": False,
                "QueryType": "Phrase",
                "SearchFeature": "browseresult",
                "SelectedCourt": "0000001111111111100001100100000000110000011111111000111111000001000100101000000000000000000000000000011101111111111111110110011101111111001100000000001111110100110111111111111111000111111111011111111110001111101110011111001011011111",
                "IsIclrContent": True,
                "packageGroupId": "1",
                "IsMootCourtAccessible": "false",
            }
        }
    else:
        data = {
            "searchDetails": {
                "SearchField": "",
                "QueryText": f"{id}",
                "RequiredRows": 50,
                "BuildTree": False,
                "QueryType": "Phrase",
                "SearchFeature": "browseresult",
                "SelectedCourt": "0000001111111111100001100100000000110000011111111000111111000001000100101000000000000000000000000000011101111111111111110110011101111111001100000000001111110100110111111111111111000111111111011111111110001111101110011111001011011111",
                "IsIclrContent": True,
                "packageGroupId": "1",
                "IsMootCourtAccessible": "false",
            }
        }

    while True:
        try:
            response = requests.post(url, headers=headers, json=data)
            return response.json()
        except:
            time.sleep(0.5)
            pass


def get_text_from_citation(ASPXAUTH, id, path):
    url = "https://www.scconline.com/HelperServices/ServicesForCourtFunctionality.asmx/GetPageData"
    headers = {
        "Content-Type": "application/json; charset=UTF-8",
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "X-Requested-With": "XMLHttpRequest",
        "Cookie": ASPXAUTH,
        "Referer": "https://www.scconline.com/Members/NoteView.aspx?enc=KDIwMDcpIDEgU0NDIDczMiYmJiYmNDAmJiYmJkJyb3dzZVBhZ2U=",
        "Origin": "https://www.scconline.com",
    }

    data = {
        "searchDetails": {
            "QueryText": f"{id}",
            "SearchField": "",
            "SelectedCourt": "",
            "QueryType": "Phrase",
            "SearchInResultQuery": [],
            "SearchWithSynStem": False,
            "UserName": "ca2a23b9cbbcaa08009d889dfc725559",
            "SearchFeature": "",
            "DOIForHighlight": "",
            "StatuesReference": "false",
        },
        "path": f"{path}",
        "subheadingOrCitation": "",
        "isHighlight": False,
        "valueForXslt": "",
        "sectionSearchingForXslt": "",
        "DisplayNameFromTree": "",
    }

    while True:
        try:
            response = requests.post(url, headers=headers, json=data)
            return response.json()
        except:
            time.sleep(0.5)
            pass


if check_login() == False:

    url = "https://www.scconline.com"
    username = "Advbharatbagla@gmail.com"
    password = "Goodfortune@2024"
    response, crisp = login_to_website(url, username, password)
    enc = response.json()["Url"].split("=")[1]
    ASPXAUTH = get_aspxauth_cookie(url, enc)

    court_types = ["  India", "International"]
    def process_data_citation(conn, ASPXAUTH, citations_onclick, ssc_id):
        for cit in citations_onclick:
            try:
                if "JTXT" in cit:
                    response = get_data_citation(ASPXAUTH, "DOI", cit)["d"]
                    pattern = r"\"Path\":\"(.*?)\.xml\""
                    match = re.search(pattern, response)

                    if match:
                        path = match.group(1).replace("\\\\", "\\") + ".xml"
                        text = ""
                        for _ in range(5):
                            try:
                                text = get_text_from_citation(ASPXAUTH, cit, path)["d"]
                                break
                            except:
                                pass
                        soup = BeautifulSoup(text, "html.parser")
                        title_element = soup.find(class_="SectionheadText")
                        text_cit = soup.prettify()
                        if title_element:
                            title = title_element.get_text(strip=True)
                        else:
                            # Use regex to extract the ssc_id from the HTML content
                            pattern = r'<div class="SectionheadText".*?>(.*?)<\/div>'
                            try:
                                match = re.search(pattern, str(text_cit))

                                if match:
                                    title = match.group(0)
                                else:
                                    title = None
                            except:
                                title = None

                        type = "STATUTE"
                        cit_id = cit

                        insert_citation(conn, title, text_cit, cit_id, type)

                        associate_citation(conn, ssc_id, cit_id)

                else:
                    response = get_data_citation(ASPXAUTH, "", cit)["d"]
                    pattern = r"\"Path\":\"(.*?)\.xml\""
                    match = re.search(pattern, response)

                    if match:
                        path = match.group(1).replace("\\\\", "\\") + ".xml"
                        text=""
                        for _ in range(5):
                            try:
                                text = get_text_from_citation(ASPXAUTH, cit, path)["d"]
                                break
                            except:
                                pass

                        soup = BeautifulSoup(text, "html.parser")
                        title_element = soup.find(class_="SectionheadText")
                        text_cit = soup.prettify()
                        if title_element:
                            title = title_element.get_text(strip=True)
                        else:
                            # Use regex to extract the ssc_id from the HTML content
                            pattern = r'<div class="SectionheadText".*?>(.*?)<\/div>'
                            try:
                                match = re.search(pattern, str(text_cit))

                                if match:
                                    title = match.group(0)
                                else:
                                    title = None
                            except:
                                title = None

                        type = "PRECEDENT"
                        cit_id = cit

                        insert_citation(conn, title, text_cit, cit_id, type)

                        associate_citation(conn, ssc_id, cit_id)
            except:
                pass


    def process_day(conn,court_type, court, subcourt,year, month, day):
        if subcourt:
            titles = get_all_case_titles_2(ASPXAUTH, court_type, str(court["key"]).split("$")[0], subcourt, day, month, year)["d"][0]["children"]
            add_to_scraped(conn, court_type, subcourt, year, month, day, True)
            for title in titles:
                for _ in range(5):
                    try:
                        xmlpath = get_xml_path(ASPXAUTH, str(title["key"]).split("$")[0])["d"]
                        # middlepanel = get_middle_panel(ASPXAUTH, xmlpath)["d"]
                        # middlepanel_data = json.loads(middlepanel)
                        pagedata = get_page_data(ASPXAUTH, xmlpath)["d"]

                        soup = BeautifulSoup(pagedata, "html.parser")

                        element = soup.find(class_="j")
                        case_no_element = soup.find(class_="caseno")

                        date_str = f"{year}-{month}-{day}"
                        date_obj = datetime.strptime(date_str, "%Y-%m-%d")

                        citations = soup.find_all(class_="citalink")

                        judge_name = element.text if element else None
                        courtname = subcourt
                        case_name = title["title"]
                        case_no = case_no_element.text if case_no_element else None
                        postgres_date = date_obj.strftime("%Y-%m-%d")
                        advocates = [e.text for e in soup.find_all(class_="advo")]
                        citations_onclick = [str(a.get("onclick")).split("'")[1] for a in citations]
                        inner_text = soup.prettify()
                        ssc_id_element = soup.find(class_="SectionheadText")
                        if ssc_id_element:
                            ssc_id = ssc_id_element.get_text(strip=True)
                        else:
                            # Use regex to extract the ssc_id from the HTML content
                            pattern = r'<div class="SectionheadText".*?>(.*?)<\/div>'
                            try:
                                match = re.search(pattern, str(inner_text))

                                if match:
                                    ssc_id = match.group(0)
                                else:
                                    ssc_id = None
                            except:
                                ssc_id = None

                        conn = connect_with_retry()
                        if ssc_id:
                            add_entry(conn, judge_name, courtname, case_name, case_no, postgres_date, advocates,
                                      citations_onclick, inner_text, ssc_id)

                            process_data_citation(conn, ASPXAUTH, citations_onclick, ssc_id)

                        conn.close()
                        break
                    except:
                        pass

        else:
            titles = get_all_case_titles_1(ASPXAUTH, court_type, str(court["key"]).split("$")[0], day, month, year)["d"][0]["children"]
            add_to_scraped(conn, court_type, str(court["key"]).split("$")[0], year, month, day, True)
            for title in titles:
                for _ in range(5):
                    try:
                        xmlpath = get_xml_path(ASPXAUTH, str(title["key"]).split("$")[0])["d"]
                        # middlepanel = get_middle_panel(ASPXAUTH, xmlpath)["d"]
                        # middlepanel_data = json.loads(middlepanel)

                        pagedata = get_page_data(ASPXAUTH, xmlpath)["d"]

                        soup = BeautifulSoup(pagedata, "html.parser")

                        element = soup.find(class_="j")
                        case_no_element = soup.find(class_="caseno")

                        date_str = f"{year}-{month}-{day}"
                        date_obj = datetime.strptime(date_str, "%Y-%m-%d")
                        citations = soup.find_all(class_="citalink")
                        judge_name = element.text if element else None
                        courtname = str(court["key"]).split("$")[0]
                        case_name = title["title"]
                        case_no = case_no_element.text if case_no_element else None
                        postgres_date = date_obj.strftime("%Y-%m-%d")
                        advocates = [e.text for e in soup.find_all(class_="advo")]
                        citations_onclick = [str(a.get("onclick")).split("'")[1] for a in citations]
                        inner_text = soup.prettify()
                        ssc_id_element = soup.find(class_="SectionheadText")
                        if ssc_id_element:
                            ssc_id = ssc_id_element.get_text(strip=True)
                        else:
                            # Use regex to extract the ssc_id from the HTML content
                            pattern = r'<div class="SectionheadText".*?>(.*?)<\/div>'
                            try:
                                match = re.search(pattern, str(inner_text))

                                if match:
                                    ssc_id = match.group(0)
                                else:
                                    ssc_id = None
                            except:
                                ssc_id =None
                        conn = connect_with_retry()
                        if ssc_id:
                            add_entry(conn, judge_name, courtname, case_name, case_no, postgres_date, advocates,
                                      citations_onclick, inner_text, ssc_id)

                            process_data_citation(conn, ASPXAUTH, citations_onclick, ssc_id)

                        conn.close()
                        break
                    except:
                        pass


    conn = connect_with_retry()
    create_scc_table(conn)
    create_citations_table(conn)
    create_scc_citations_table(conn)
    create_scraped_table()
    # Get the number of available CPU cores
    max_threads = 100
    threads = []
    semaphore = threading.Semaphore(max_threads)  # create a semaphore with a max limit of max_threads


    def process_day_threaded(conn,court_type, court, subcourt, year, month, day):
        process_day(conn,court_type, court, subcourt, year, month, day)


    def process_day_threaded_safe(*args):
        try:
            process_day_threaded(*args)
        except Exception as e:
            pass
            # print(f"Error in thread {threading.current_thread().name}: {e}")
        finally:
            semaphore.release()  # release the semaphore once the thread is done

 # ["d"][0]["children"]
    for index in range(0,1):
        courts = get_all_courts(ASPXAUTH, court_types[index])["d"][0]["children"]
        for court in courts:
            print(court)
            data = get_subcourts(ASPXAUTH, court_types[index], str(court["key"]).split("$")[0])
            if not data.strip():
                print("Received empty data")
                continue  # Skip to the next iteration if data is empty
            try:
                data = json.loads(data)
                if data["d"]==None:
                    pass
                else:
                    subcourts = data["d"][0]["children"]
            except (KeyError, TypeError, IndexError) as e:
                print(f"Error accessing subcourts: {e}")
                continue
            
            print('subcourts', subcourts[0])

            if subcourts[0]['level'] == 'Year':
                years = [int(year['key'].split("$")[0]) for year in subcourts]
                if not years:
                    continue  # Skip to the next court if no years are found

                for year in years:
                    print('years', year)
                    month = 12
                    current_day = 31
                    if year==2024:
                        progress_bar = tqdm(total=(31 * 12),
                                            desc=f'Processing for the court type {court_types[index]} and the court name {str(court["key"]).split("$")[0]} in year {year}')
                        last_completed_day = get_last_completed_day(court_types[index], str(court["key"]).split("$")[0], year)
                        for month in range(1, month + 1):
                            for day in range(1, current_day + 1):
                                if last_completed_day:
                                    if day > last_completed_day[1] and month > last_completed_day[0]:
                                        semaphore.acquire()  # acquire the semaphore before starting a new thread
                                        thread = threading.Thread(target=process_day_threaded_safe,
                                                                args=(conn, court_types[index], court, None, year, month, day))
                                        thread.start()
                                        threads.append(thread)
                                        progress_bar.update(1)
                                    else:
                                        progress_bar.update(1)
                                else:
                                    semaphore.acquire()  # acquire the semaphore before starting a new thread
                                    thread = threading.Thread(target=process_day_threaded_safe,
                                                            args=(conn, court_types[index], court, None, year, month, day))
                                    thread.start()
                                    threads.append(thread)
                                    progress_bar.update(1)

                        progress_bar.close()

            elif subcourts[0]['level'] == 'YearRange':
                for yearrange in subcourts:
                    min_year = int(str(yearrange['key'].split("$")[0]).split(" - ")[0])
                    max_year = int(str(yearrange['key'].split("$")[0]).split(" - ")[1])

                    if min_year==1950:
                        if min_year > max_year:
                            continue  # Skip if the year range is invalid

                        progress_bar = tqdm(total=(((max_year + 1) - min_year) * 31 * 12),
                                            desc=f'Processing for the court type {court_types[index]} and the court name {str(court["key"]).split("$")[0]} in year range {str(yearrange["key"].split("$")[0])}')
                        for year in range(min_year, max_year + 1):
                            month = 12
                            current_day = 31

                            last_completed_day = get_last_completed_day(court_types[index], str(court["key"]).split("$")[0], year)
                            for month in range(1, month + 1):
                                for day in range(1, current_day + 1):
                                    if last_completed_day:
                                        print('last', year, (day, month), last_completed_day)
                                        if day > last_completed_day[1] and month > last_completed_day[0]:
                                            semaphore.acquire()  # acquire the semaphore before starting a new thread
                                            thread = threading.Thread(target=process_day_threaded_safe,
                                                                    args=(conn, court_types[index], court, None, year, month, day))
                                            thread.start()
                                            threads.append(thread)
                                            progress_bar.update(1)
                                        else:
                                            progress_bar.update(1)
                                    else:
                                        semaphore.acquire()  # acquire the semaphore before starting a new thread
                                        thread = threading.Thread(target=process_day_threaded_safe,
                                                                args=(conn, court_types[index], court, None, year, month, day))
                                        thread.start()
                                        threads.append(thread)
                                        progress_bar.update(1)

                        progress_bar.close()

            else:
                for sub in subcourts:
                    subcourt = str(sub['key'].split("$")[0])
                    print(subcourt,"Court: ",str(court["key"]).split("$")[0],"type: ",court_types[index])
                    years = get_year_range_for_subcourt(ASPXAUTH, str(court["key"]).split("$")[0], subcourt, court_types[index])["d"][0]["children"]
                    print(years)
                    
                    if not years:
                        continue  # Skip to the next subcourt if no years are found

                    if years[0]['level'] == 'Year':
                        years = [int(year['key'].split("$")[0]) for year in years]
                        if not years:
                            continue  # Skip if there are no valid years

                        for year in years:
                            month = 12
                            current_day = 31
                            print(year)
                            if year==2021:
                                progress_bar = tqdm(total=(31 * 12),
                                                    desc=f'Processing for the court type {court_types[index]} and the court name {str(court["key"]).split("$")[0]} for the subcourt {subcourt} in year {year}')
                                last_completed_day = get_last_completed_day(court_types[index], subcourt, year)
                                for month in range(1, month + 1):
                                    for day in range(1, current_day + 1):
                                        if last_completed_day:
                                            if day > last_completed_day[1] and month > last_completed_day[0]:
                                                semaphore.acquire()  # acquire the semaphore before starting a new thread
                                                thread = threading.Thread(target=process_day_threaded_safe,
                                                                        args=(conn, court_types[index], court, subcourt, year, month, day))
                                                thread.start()
                                                threads.append(thread)
                                                progress_bar.update(1)
                                            else:
                                                progress_bar.update(1)
                                        else:
                                            semaphore.acquire()  # acquire the semaphore before starting a new thread
                                            thread = threading.Thread(target=process_day_threaded_safe,
                                                                    args=(conn, court_types[index], court, subcourt, year, month, day))
                                            thread.start()
                                            threads.append(thread)
                                            progress_bar.update(1)

                                progress_bar.close()

                    elif years[0]['level'] == 'YearRange':
                        for yearrange in years:
                            min_year = int(str(yearrange['key'].split("$")[0]).split(" - ")[0])
                            max_year = int(str(yearrange['key'].split("$")[0]).split(" - ")[1])
                            
                            if min_year==1950:
                                if min_year > max_year:
                                    continue  # Skip if the year range is invalid

                                progress_bar = tqdm(total=(((max_year + 1) - min_year) * 31 * 12),
                                                    desc=f'Processing for the court type {court_types[index]} and the court name {str(court["key"]).split("$")[0]} for the subcourt {subcourt} in year range {str(yearrange["key"].split("$")[0])}')
                                for year in range(min_year, max_year + 1):
                                    month = 12
                                    current_day = 31
                                    last_completed_day = get_last_completed_day(court_types[index], subcourt, year)
                                    for month in range(1, month + 1):
                                        for day in range(1, current_day + 1):
                                            if last_completed_day:
                                                if day > last_completed_day[1] and month > last_completed_day[0]:
                                                    semaphore.acquire()  # acquire the semaphore before starting a new thread
                                                    thread = threading.Thread(target=process_day_threaded_safe,
                                                                            args=(conn, court_types[index], court, subcourt, year, month, day))
                                                    thread.start()
                                                    threads.append(thread)
                                                    progress_bar.update(1)
                                                else:
                                                    progress_bar.update(1)
                                            else:
                                                semaphore.acquire()  # acquire the semaphore before starting a new thread
                                                thread = threading.Thread(target=process_day_threaded_safe,
                                                                        args=(conn, court_types[index], court, subcourt, year, month, day))
                                                thread.start()
                                                threads.append(thread)
                                                progress_bar.update(1)

                                progress_bar.close()


    for thread in threads:
        thread.join()

    print("done")