are these methods
nonLocalBrokerMapCREATE(nonLocalMapFile="nonLocalBrokersToEIN")
localBrokerRevertMapCREATE
Create_Broker_Comm_Fee()

present in
print("\nSource 01impute_lob.py", dot20)

notWelfareInsTerms = [
    "TAX PREP",
    "SOFTWARE",
    "ASSET M",
    "ASSET SER",
    "BANK",
    "LAW OFFICE",
    "LABOR ORG",
    "LABOR UN",
    "COMMON AFFIL",
    "ACCOUNTING",
    "ACCOUNTAN",
    "DATA ANAL",
    "HUMAN MANAGEMENT",
    "DATA SERV",
    "INVEST",
    "ANDREA",
    "FABIAN",
    "GHS DATA",
    "CONTRACTOR",
    "AFFILIAT",
    "SURVEY",
    "NORTHGATE",
    "EMPLOYEE OF PL",
    "CONTRIB EMP",
    "SPONSOR",
    "DATA SER",
    "MICHAEL",
    "BRIAN",
    "WAKEFIELD",
    "ASSOCIATES",
    "MCDONALD",
    "DONNELLEY",
    "CAPITAL MA",
    "TRUSTEE",
]

notMedTerms = notWelfareInsTerms + [
    "HERITAGE",
    "AUXILLARY",
    "ACTUAR",
    "CASE M",
    "AUXILIARY",
    "AMERICAN HER",
    "TIC INTERNATIONAL",
    "MBWWA SERVICES",
    "EBS MASTER",
    "EPK AND",
    "HEALTH ADVOCATE",
    "CONTINENTAL AMERICAN INS",
    "CANADA",
    "ASSET",
    "UNRELATED",
    "NOT RELATE",
    "FSA ADMIN",
    "UTILIZATION REV",
    "UTIL REV",
    "FSA HRA",
    "DISEASE M",
    "OTHER FEES",
    "FSA COBR",
    "WEIGHT WATCH",
    "CANADA",
    "STATE PENN",
]

volLifeStrings = [
    "VOLUNTARY LIFE",
    "VOL LIFE",
    "MENTAL LIFE",
    "SUPP LIFE",
    "DEPENDENT LIFE",
    "DEP LIFE",
    "LIFE VOL",
    "LIFE SUPP",
    "VOL CHILD",
    "UNIVERSAL LIF",
    "SPOUSE LIF",
    "OPTIONAL LIFE",
    "GVUL",
    "VOLUNTARY GROUP TERM",
    "VOL GROUP TERM",
    "SUPPLEMENTAL TERM LIFE",
]

volStringsGeneric = [
    "VOLUNTARY",
    "VOL BEN",
    "SUPPLEMENTAL IND",
    "FIXED IND",
    "SUPP BEN",
    "SUPP PRO",
    "SUPP COV",
    "LEMENTAL HE",
    "LEMENTAL BE",
    "LEMENTAL COV",
    "LEMENTAL PRO",
    "SPECIAL RISK",
    "WORKPLACE",
    "WORK PLACE",
    "WORK SITE",
    "VOLUNTARY GR",
    "VOLUNTARY PL",
    "VOL PL",
    "MEDICAL GAP",
    "MED BRIDGE",
    "MEDICAL SUP",
    "MEDICAL BRIDGE",
    "MED SUP",
    "MEDICAL LEAV",
    "MENTAL MED",
    "MEDIGAP",
    "ABROAD",
    "COUNTRY MED",
    "EXPAT",
    "GLOBAL MED",
    "WORKSITE",
    "INDIVIDUAL PROD",
    "SUPPLEMENTAL INS",
    "WORKERS COMP",
    "INJURY",
]

servicesNotInsKeywords = [
    "TELEDOC",
    "TELADOC",
    "TELEMED",
    "800MD",
    "TELEPHONE",
    "TELEHEALTH",
    "ADVOCA",
    "CASE M",
    "HEALTH ADVIS",
    "ADVISING",
    "SITE CLINIC",
    "SCREENING",
    "HEALTH CLINIC",
    "IDENTITY THE",
    "PREVENTIVE",
    "KIDNAP",
    "UTILIZATION REVIEW VEND",
]

import os


def impute_lob(y, inFile="welfSchedAll", 
               outFile="welfImputeLob", SINK=True):
   
    fn = "imputeLob"
    start_time_impute_lob = time.time()
    fnStart = datetime.now()
    rulesDt = None

    # closeAllConnections()
    
    loadEndogenousParamsThisYear(y=y, DETAILS=False)

    print("\n\nStart impute_lob for ", y, ". Load ", inFile, " from pitByYear/", y)

    # Load dataset and function intro stock functions
    init = pd.read_pickle(pitByYearDir + str(y) + "/" + inFile + ".pkl")

 
    init["Year"] = y
    #  init=init[init.TOT_PART_BOY>10000]
    beginobs = len(init)
    begvar = list(init.columns)

    print("Observations in welfSchedAll", dot20, beginobs)
    # init.DEFINED_CONT.value_counts(dropna=False)

    # removal of original filings where amended records are present.
    
    rule1 = "RemoveOriginal_AmendedAvailable : DeDuplication1"
    init["KEY"] = (
        init["EIN"].astype(str)
        + "_"
        + init["PN"].astype(str)
        + "_"
        + init["Year"].astype(str)
        + "_"
        + init["CTR_NUM"].astype(str)
        + "_"
        + init["WelfCodes"].astype(str)
    )
    amended_key_list = init[init["AMENDED"] == 1][["KEY"]].drop_duplicates()

    init["FLAG_rule1"] = np.where(
        (init["KEY"].isin(amended_key_list["KEY"])) & (init["AMENDED"] == 0), 1, 0
    )

    flag_rule1_Keys = init[init["FLAG_rule1"] == 1][["KEY"]].drop_duplicates()
    init["FLAG_rule1"] = np.where(
        init["KEY"].isin(flag_rule1_Keys["KEY"]), 1, init["FLAG_rule1"]
    )

    print("\nDistribution of records flagged ", "\n")
    print(pd.crosstab(init["FLAG_rule1"], columns="count"))
    init["Rules"] = np.where(
        (init["FLAG_rule1"] == 1) & (init["AMENDED"] == 1),
        init["Rules"] + " Remove_Amendment_Rule1; ",
        init["Rules"],
    )
    
 
    init["FLAG_rule1"] = np.where(
        (init["FLAG_rule1"] == 1) & (init["AMENDED"] == 1), 0, init["FLAG_rule1"]
    )

    init = init[init["FLAG_rule1"] == 0]
    print(
        "\nRecords dropped due to amended filing available _rule 1 = ",
        beginobs - init.shape[0],
    )

    init = init.drop(columns=["FLAG_rule1", "KEY"])

    rule2 = "RemoveOriginal_AmendedAvailable : DeDuplication2"

    # create a key column by concatenating EIN, PLAN_NAME, Year, CTR_NUM and WelfCodes
    init["KEY"] = (
        init["EIN"].astype(str)
        + "_"
        + init["PLAN_NAME"].astype(str)
        + "_"
        + init["Year"].astype(str)
        + "_"
        + init["CTR_NUM"].astype(str)
        + "_"
        + init["WelfCodes"].astype(str)
    )

    # create a list of keys for records with AMENDED == 1
    amended_key_list = init[init["AMENDED"] == 1]["KEY"].unique()

    # create a flag column for records with keys in amended_key_list and AMENDED == 0
    
    init["FLAG_rule2"]=np.where((init.KEY.isin( amended_key_list))
                                & (init.AMENDED==0),1,0)


    # create a list of keys for records with FLAG_rule2 == 1
    flag_rule2_Keys = init[init["FLAG_rule2"] == 1]["KEY"].unique()

    # update the flag column for records with keys in flag_rule2_Keys
    
    init["FLAG_rule2"] = np.where(init['KEY'].isin(flag_rule2_Keys),1,init["FLAG_rule2"] )
     
    
    # print the distribution of records flagged
    print("\nDistribution of records flagged \n")
    print(init["FLAG_rule2"].value_counts())

    # update the rules column for records with FLAG_rule2 == 1 and AMENDED == 1
    init["Rules"] = np.where(
        (init["FLAG_rule2"] == 1) & (init["AMENDED"] == 1),
        init["Rules"] + " Remove_Amendment_Rule2; ",
        init["Rules"],
    )

    init=init.reset_index(drop=True)
    # update the flag column for records with FLAG_rule2 == 1 and AMENDED == 1
    init.loc[(init["FLAG_rule2"] == 1) & (init["AMENDED"] == 1), "FLAG_rule2"] = 0

    # drop the records with FLAG_rule2 == 1
    init = init[init["FLAG_rule2"] == 0]
    print(
        "\nRecords dropped due to amended filing available _rule 2 and _rule1= ",
        beginobs - len(init),
    )

    # drop the flag and key columns
    init.drop(["FLAG_rule2", "KEY"], axis=1, inplace=True)

    if "PLAN_NAME" in init.columns:
        init = init.rename(columns={"PLAN_NAME": "Plan_Name"})

    txt = "\nStart" + fn + "for" + str(y) + "; Obs in A+C:" + dot20 + str(init.shape[0])
    print(txt)

    if SINK:
        logDir = logwd + str(y) + "/"
        logFile = logDir + fn + "_" + ymd + ".txt"
        if not os.path.exists(logDir):
            os.makedirs(logDir)
            print("  Created", logDir)
        print("\nSink console output to ", logFile)
        #    sys.stdout = open(logFile, "w")
        print(txt)

    rule = "DropNoCarrierNoEIN"
    init["DROP"] = np.where(
        (init["Carrier_Orig"] == "") & (init["Carrier_EIN"] == ""), 1, 0
    )
    rulesDt = ruleCapt(
        DROP=True,
        sub=init.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=75)

    init = init[init["DROP"] == 0]

    pctMed = init["MEDICAL_PN"].mean()
    if pctMed < 0.5:
        raise Exception(
            "\nPct welfare obs that are medical plans is only" + str(pctMed)
        )
    else:
        print("\nPct of welfare obs that are MEDICAL_PN " + str(pctMed))
        
 

    # Check if "TOTAL_PART" is in the column names of init
    if "TOTAL_PART" in init.columns:
        init.rename(columns={"TOTAL_PART": "Total_Part"}, inplace=True)

    # Create a new column "PART_MAX" with the values of "Part_EOY"
    init["PART_MAX"] = init["Part_EOY"]
    print("\nPct of with PART_MAX < 10 ", len(init[init["PART_MAX"] < 10]) / len(init))

    # Update the values of "PART_MAX" based on some conditions
    
    init['PART_MAX']=np.where( (init.Part_EOY<25) & (init['Total_Part']>25),
                              init.Total_Part,init.PART_MAX)
    
    init['PART_MAX'].sum()
    
    init['PART_MAX']=np.where( (init.PART_MAX<25) & (init['TOT_PART_BOY']>25),
                              init.TOT_PART_BOY,init.PART_MAX)
    

    # print the percentage of records with PART_MAX < 10
    print(
        "\nAfter adjust. % with PART_MAX < 10 for",
        y,
        dot20,
        len(init[init["PART_MAX"] < 10]) / len(init),
    )

    # create a Duration column if it does not exist
    if "Duration" not in init.columns:
        init["Duration"] = pd.to_numeric(
            pd.to_datetime(init["PLAN_YEAR_END_DATE"], errors="coerce")
            - pd.to_datetime(init["PLAN_YEAR_BEGIN_DATE"], errors="coerce")
        )

    # Define a rule as a string
    rule = "\nUpdate to Duration_Ctr if available on Sched A"

    # Create a new column "Duration_Ctr" with the difference between two date columns
    init["Duration_Ctr"] = (
        pd.to_datetime(init["POLICY_TO_DATE"], errors="coerce")
        - pd.to_datetime(init["POLICY_FROM_DATE"], errors="coerce")
    ).dt.days

   
    # Replace the missing values in "Duration_Ctr" with 364
    init["Duration_Ctr"].fillna(364, inplace=True)
    init['POLICY_TO_DATE'] = pd.to_datetime(init['POLICY_TO_DATE'], errors='coerce')
    init['Duration_Ctr'] = np.where(init['POLICY_TO_DATE'].dt.year >= 3000, 364, init['Duration_Ctr'])
    init['Duration_Ctr'] = np.where(init['POLICY_TO_DATE'].dt.year <= 1800, 364, init['Duration_Ctr'])
    init['POLICY_TO_DATE'] = init['POLICY_TO_DATE'].dt.date
    init['POLICY_TO_DATE'] = init['POLICY_TO_DATE'].replace(pd.NaT, '').fillna('')
    init['POLICY_TO_DATE'] = init['POLICY_TO_DATE'].astype('object')
    init['POLICY_TO_DATE'] = init['POLICY_TO_DATE'].fillna("")
    
    # Create a new column "FL" with 1 or 0 based on a condition
    init["FL"] = np.where((init["Duration_Ctr"] != init["Duration"]) & (init["Sch"] == "A"), 1, 0)
    print(rule, y, " ", init["FL"].sum())

    # Update the values of "Duration" based on "FL"
    init["Duration"] = np.where(init["FL"] == 1, init["Duration_Ctr"], init["Duration"])
    # Add a temporary column "PMPM_EST" based on some calculations
    init["PMPM_EST"] = np.where(
        init["Members"] < 5, 0, lasj_round(init["Premiums"] / (12 * init["Members"]))
    )
    print(init["PMPM_EST"].describe())
    print(init["PMPM_EST"].sum())
    # Add a temporary column "PMPM_MED_FI" based on some conditions
    init["PMPM_MED_FI"] = np.where(
        init["Medical"] == 1,
        500,
        np.where(
            init["Dental"] == 1,
            50,
            np.where(
                (init["LTD"] == 1) | (init["STD"] == 1) | (init["Life"] == 1), 20, 0
            ),
        ),
    )

    init["PMPM_MED_FI"] = np.where(
        (init["PMPM_MED_FI"] == 0) & (init["Vision"] == 1),
        10,
        np.where(init["Stoploss"] == 1, 80, init["PMPM_MED_FI"]),
    )
    init["PMPM_MED_FI"] = lasj_round(
        init["PMPM_MED_FI"] - 0.05 * init["PMPM_MED_FI"] * (2018 - y)
    )
    print(pd.crosstab(index=init["PMPM_MED_FI"], columns="count"))

    # Add a temporary column "RATIO" based on some calculations
    init["RATIO"] = np.where(
        init["PMPM_MED_FI"] < 0.5,
        1,
        lasj_round(init["PMPM_EST"] / init["PMPM_MED_FI"], 1),
    )
    print(init["RATIO"].describe())

    # Define another rule as a string
    rule = "Adjust bad durations - set to 364 if PMPM looks OK"

    # Update the values of "FL" based on some conditions
    init["FL"] = np.where(
        (init["Duration"] < 50)
        & (init["RATIO"] > 0.5)
        & (init["RATIO"] < 2)
        & (init["PMPM_MED_FI"] > 1)
        & (init["Sch"] == "A"),
        1,
        0,
    )

    # Print the sum of "FL"
    print(init["FL"].sum())

    # Print the end message with the rule, y and the sum of "FL"
    print(
        "\nEnd",
        rule,
        "for",
        y,
        ". Obs to set to 364 b/c look like normal pmpm ",
        init["FL"].sum(),
    )

    # Update the values of "Duration" based on "FL"
    init["Duration"] = np.where(init["FL"] == 1, 364, init["Duration"])

    # Update the values of "Rules" by concatenating strings based on "FL"
    init["Rules"] = np.where(
        init["FL"] == 0, init["Rules"], init["Rules"] + " DurationSetTo364bcPMPM; ")
    
      
  # Addition before Jun-July 23 refresh
    rule="If Duration is -1"
    init['FL']=np.where(init.Duration == -1, 1, 0)
    print("\nEnd ", rule, y, " forced to 5 from 0-5 ", sum(init.FL))
    init['Duration']=np.where(init.FL==1, 364 ,init.Duration)
    init['Rules']=np.where(init.FL == 0,init.Rules,
                      init.Rules+" DurationSetTo364InsteadOfMinusOne; ")
    init['POLICY_TO_DATE'] = init['POLICY_TO_DATE'].replace(pd.NaT, '').fillna('')
   # init = init.fillna("", subset=init.columns)
    rulesDt = ruleCapt(sub = init.copy(), DROP = True, rulesDt = rulesDt, 
                         rule = rule, Module = fn,Yr = y, row=200)

    rule = "If Duraiton still very veyr low set minimum duration of 5"
    # Create a new column "FL" in init based on a condition
    init["FL"] = np.where(init["Duration"] < 5, 1, 0)

    # Print the end message with the rule, y and the sum of "FL"
    print("\nEnd ", rule, y, " forced to 5 from 0-5 ", init["FL"].sum())

    # Update the values of "Duration" based on "FL"
    init["Duration"] = np.where(init["FL"] == 1, 5, init["Duration"])

    # Update the values of "Rules" by concatenating strings based on "FL"
    init["Rules"] = np.where(
        init["FL"] == 0, init["Rules"], init["Rules"] + " DurationSetTo5; ")
    

    # Call the ruleCapt function with some arguments
    rulesDt = ruleCapt(
        DROP=True,
        sub=init.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=119,
    )

    # Update the values of "FL" based on another condition
    init["FL"] = np.where(init["Duration"] < 2, 1, 0)
    #init.Duration.value_counts(dropna=False).to_csv('dur1.csv')             

    # Check if carStdwelf exists and load it if not
    if globals().get("carStdwelf") is None:
        global carStdwelf
        loadCarStd(WELF=True, RET=False)
        carStdwelf["Carrier_EIN"] = (
            carStdwelf.Carrier_EIN.astype(str).str.split(".",expand=True)[0]
        )
       
        for col in ["Carrier_INITIAL", "Carrier_String"]:
            # For these cols, avoid removing . entirely because values can be names with .
            loc_condition = (
                carStdwelf[col]
                .astype(str)
                .replace("\.0", "", regex=True)
                .str.isnumeric()
            ) & (carStdwelf[col].notna())
            carStdwelf.loc[loc_condition, col] = (
                carStdwelf.loc[loc_condition, col].astype(int).astype(str)
            )
    # Define a list of columns to add from carStdwelf
    addCarVars = ["Carrier", "Carrier_String"]
   
    # Create a subset of carStdwelf with unique values of some columns
    carMap = carStdwelf[
        addCarVars + ["Carrier_INITIAL", "Carrier_EIN"]
    ].drop_duplicates()

    # Print the dimensions of carMap
    print(carMap.shape)
    # Create a new column "CT" in carMap with the count of "Carrier" by some groups
    carMap["CT"] = carMap.groupby(["Carrier_INITIAL", "Carrier_EIN"])[
        "Carrier"
    ].transform("count")

    # Create a new column "Stand" in carMap based on a condition
    carMap["Stand"] = np.where(carMap["Carrier"].isin(carrier_exact["Carrier"]), 1, 0)

    # Create a new column "DROP" in carMap based on some conditions
    carMap["DROP"] = np.where((carMap["Stand"] == 0) & (carMap["CT"] > 1), 1, 0)

    # Print the number of rows to drop from carMap
    print("\nDrop from carrier map b/c multi not stand ", y, " ", carMap["DROP"].sum())

    # Filter out the rows with "DROP" equal to 1 from carMap
    carMap = carMap[carMap["DROP"] == 0]
    carMap.drop(columns="DROP", inplace=True)

    # Update the values of "CT" in carMap with the count of "Carrier" by some groups
    carMap["CT"] = carMap.groupby(["Carrier_INITIAL", "Carrier_EIN"])[
        "Carrier"
    ].transform("count")

    # Create a subset of carMap with some conditions and sort by a column
    te = (
        carMap[(carMap["CT"] > 1) & (carMap["Carrier_EIN"] != "")]
        .drop_duplicates()
        .sort_values(by="Carrier_EIN")
    )

    # TODO: uncomment below code and fix this
    print(carMap.head())
    print(te.head())
    print("About to go into failure")
    # # Check if te is not empty and assign it to stopped and raise an error
    # if len(te) > 0:
    #     stopped = te.copy()
    #     raise Exception(len(te) / 2, " Dup initial X Carrier_EIN obs @135 for ", y)

    # Print the number of rows and unique values in carMap
    print(carMap.columns)
    # # TODO: fix this. INITIAL doesnt exist
    # print(
    #     "\nObs in carrier STAND file:",
    #     len(carMap),
    #     " uq Carrier_Origs ",
    #     len(carMap["INITIAL"].unique()),
    # )
    # Drop the column "Stand" from carMap
    carMap.drop(columns="Stand", inplace=True)

    # Create a new column "LTC" in init based on a condition
    init["LTC"] = np.where(
        (init["Plan_Name"].str.contains("LONG TERM C"))
        | (init["Plan_Name"].str.contains("LT C")),
        1,
        0,
    )

    # Update the values of "Rules" by concatenating strings based on "LTC"
    init["Rules"] = np.where(
        init["LTC"] == 0,
        init["Rules"],
        init["Rules"] + " LOBAssign_LTC_PlanName_Keyword; ",
    )

    # Create a new column "Funding" in init based on "Sch"
    init["Funding"] = np.where(init["Sch"] == "A", "FI", "SI")

    # Early funding rules before drop other
    # Flag the large MEWA cause going to drop

    # Create a new column "TENPLUS" in init based on a condition
    init["TENPLUS"] = np.where(init["WelfCodes"].str.upper().str.contains("4T"), 1, 0)
    print("\nTENPLUS ", init["TENPLUS"].mean())
    print("\nTENPLUS ", init["TENPLUS"].sum())
    # Create a new column "MEWA" in init based on a condition
    init["MEWA"] = np.where(init["PLAN_ENTITY"].isin([3, "3"]), 1, 0)
    print("\nMEWA ", init["MEWA"].mean())

    # Create a new column "DFE" in init based on a condition
    init["DFE"] = np.where(init["PLAN_ENTITY"].isin([4, "4"]), 1, 0)
    print("\nDFE ", init["DFE"].mean())

    # Create a new column "MULTIEMP" in init based on a condition
    init["MULTIEMP"] = np.where(init["PLAN_ENTITY"].isin([1, "1"]), 1, 0)
    print("\nMultiemployer/taft harltey ", init["MULTIEMP"].mean())

    # PLAN_ENTITIES: 1 = multiemployer=taft hartley = collective bargaining; 2 = single employer; 3 = MEWA; 4 = DFE

    # Add the initial OtherFeatures column
    init["OtherFeatures"] = np.where(init["TENPLUS"] == 1, "TENPLUS; ", "")

    init["OtherFeatures"] = np.where(
        init["MEWA"] == 1, init["OtherFeatures"] + "MEWA; ", init["OtherFeatures"]
    )

    init["OtherFeatures"] = np.where(
        init["MULTIEMP"] == 1,
        init["OtherFeatures"] + "MultiemployerTaftHartley; ",
        init["OtherFeatures"],
    )

    init["OtherFeatures"] = np.where(
        init["Union"] == 1, init["OtherFeatures"] + "Union; ", init["OtherFeatures"]
    )

    init["OtherFeatures"] = np.where(
        init["DFE"] == 1, init["OtherFeatures"] + "DFE; ", init["OtherFeatures"]
    )

    rule = "SIReclass_TenPlus; "
    Text = "Assume that TENPLUS plans are SI if med or dnetal."
    print("\nObs with ten-plus-employer-pn: ", (init["TENPLUS"] == 1).sum(), dot20)

    # Create a new column "FL" in init based on some conditions
    init["FL"] = np.where(
        (
            (init["Medical"] == 1)
            | (init["Dental"] == 1)
            | (init["Vision"] == 1)
            | (init["STD"] == 1)
        )
        & (init["Sch"] == "A")
        & (init["TENPLUS"] == 1)
        & (init["Part_EOY"] > siFTEFloor)
        & (init["Members"] > 1.3 * siFTEFloor),
        1,
        0,
    )
    print("\nCaught by ", rule, y, " ", init["FL"].sum())

    # Update the values of "Funding" based on "FL"
    init["Funding"] = np.where(init["FL"] == 1, "SI", init["Funding"])
    init["Rules"] = np.where(init["FL"] == 0, init["Rules"], init["Rules"] + rule)
    

    # Call the ruleCapt function with some arguments
    rulesDt = ruleCapt(
        DROP=True,
        sub=init.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=180,
    )

    rule = "SIreclass_MEWA_or_multiemployer; "

    # Update the values of "FL" in init based on some conditions
    init["FL"] = np.where(
        (init["Sch"] == "A")
        & ((init["MEWA"] == 1) | (init["MULTIEMP"] == 1))
        & (init["Part_EOY"] > 2 * siFTEFloor)
        & (
            (init["Medical"] == 1)
            | (init["Dental"] == 1)
            | (init["STD"] == 1)
            | (init["Vision"] == 1)
        ),
        1,
        0,
    )
    print("\nCaught by ", rule, y, " ", init["FL"].sum())

    # Update the values of "Funding" based on "FL"
    init["Funding"] = np.where(init["FL"] == 1, "SI", init["Funding"])
    init["Rules"] = np.where(init["FL"] == 0, init["Rules"], init["Rules"] + rule)

    # Call the ruleCapt function with some arguments

    rulesDt = ruleCapt(
        DROP=True,
        sub=init.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=194,
    )

    # Print the number of variables and observations in init
    print("\nTot variabls now ", init.shape[1], " and obs ", init.shape[0])

    # Keep only the variables in keepVarsForRules and some others
    # NOTE: I was seeing that later, MEDICAL_PN is needed but wasnt kept because of case issue
    # Will add upper version of the vars to keep here.
    keepVars = list(set(keepVarsForRules).intersection(set(init.columns)))
    
    keepVars = keepVars + ["FORM_ID"]
    print("keepVars on line 719 for dt:", keepVars)

    # Check if FORM_ID is in keepVars
    if "FORM_ID" not in keepVars:
        raise Exception("\nFORM ID missing in year ", y)
    print(" and tot vars to keep ", len(keepVars))

    # Check if RTD_SEP_PART_RCVG is in keepVars
    if "RTD_SEP_PART_RCVG" not in keepVars:
        print("\n!! RTD_SEP_PART_RCVG missing from keepVars!!")

    # Subset init by keepVars
    dt = init[keepVars]

    if "MEDICAL_PN" not in dt.columns:
        print("ERROR. need this for later. ensure it's here now")
        raise ValueError

    dt["Carrier_EIN"] = dt["Carrier_EIN"].astype(str).str.split(".").str[0]
    dt["Carrier_EIN"] = np.where(dt["Carrier_EIN"] == "nan", "", dt["Carrier_EIN"])
    carMap["Carrier_EIN"] = np.where(
        carMap["Carrier_EIN"] == "nan", "", carMap["Carrier_EIN"]
    )

    # carMap['Carrier_EIN'] = carMap['Carrier_EIN'].astype(str)
    carMap = carMap.drop_duplicates()
    

    # Merge dt with carMap by Carrier_EIN and Carrier_Orig/Carrier_INITIAL
    dt = pd.merge(
        dt,
        carMap,
        left_on=["Carrier_EIN", "Carrier_Orig"],
        right_on=["Carrier_EIN", "Carrier_INITIAL"],
        how="left",
    )
    

    print("dt @748 after merge with carMap", dt.head(3))
    # Create map_df of Carrier_String and Carrier for EIN == 1
    map_df = carrier_exact[carrier_exact["EIN"] == 1][["Carrier_String", "Carrier"]].drop_duplicates()

    # Create a flag for Carrier not in carrier_exact and Carrier_EIN in map_df
    dt["FL"] = (~dt["Carrier"].isin(carrier_exact["Carrier"])) & (
        dt["Carrier_EIN"].isin(map_df["Carrier_String"])
    ).astype(int)
    print("\nObs to refresh standardization @170 ", dt["FL"].sum())

    # Rename Carrier to XWALK in map_df
    map_df = map_df.rename(columns={"Carrier": "XWALK"})
    print("map_df columns @763", map_df.columns)
    # Merge dt with map_df by Carrier_EIN and Carrier_String
    # TODO: NOTE: with this merge, we now have two columns named Carrier_string.
    # NOTE: 1 from carMap and one from map_df. not sure which is the proper one.
    # For now, I'll set _x as the suffic for x and leave the map_df one
    map_df.columns
    dt.Carrier_String.value_counts(dropna=False)
    
    dt = pd.merge(
        dt,
        map_df,
        left_on="Carrier_EIN",
        right_on="Carrier_String",
        how="left",
        suffixes=("", "_y"),
    )
    dt = dt.drop(["Carrier_String_y"], axis=1)

    print("dt columns @776 after merge wtih map_df", dt.columns)
    print("na in dt column at 777", dt.isna().sum())

    # Replace missing values in XWALK with empty strings
    dt["XWALK"] = dt["XWALK"].fillna("")

    # Update the flag for XWALK not empty
    dt["FL"] = (dt["FL"] == 1) & (dt["XWALK"] != "").astype(int)
    print(" Update std ", dt["FL"].sum())

    # Replace Carrier with XWALK if flag is 1
    dt['Carrier']=np.where(dt['FL']==1,dt.XWALK,dt.Carrier)
    
    # Drop XWALK column
    dt = dt.drop(columns=["XWALK"])
    # dt.to_csv('temp.csv')
    # Set missing values to empty strings for Carrier_String and Carrier
    setNABlank = ["Carrier_String", "Carrier"]
    for v in setNABlank:
        dt[v] = dt[v].fillna("")
    print("na in dt column at 798 before checkForBads", dt.isna().sum())
    # Replace Carrier with Carrier_Orig if Carrier is empty
    
    dt['Carrier']=np.where(dt.Carrier=="",dt.Carrier_Orig,dt.Carrier)

    # Drop junkVars columns from dt
    print("junkVars", junkVars)
    print("dt col", dt.columns)
    # NOTE: changed junkVars to upper case
    dt_to_drop = set([x.upper() for x in junkVars]).intersection(set(dt.columns))
    print("dt_to_drop", dt_to_drop)
    dt = dt.drop(columns=list(dt_to_drop))

    # Check for bad values in dt
    # NOTE: fix this. There are nulls entirely in columns; Carrier_String_x , Carrier_INITIAL, CT (all from (from carMap))
    dt = dt.drop(columns=["Carrier_INITIAL"])

    print("na in dt column at 809 before checkForBads", dt.isna().sum())
    checkForBads(dtCheck=dt.copy(), txt="@230 after left join to Stand Carrier", y=y)

    # Create a flag for Carrier not in carrier_exact
    dt["CarrierNotStd"] = (~dt["Carrier"].isin(carrier_exact["Carrier"])).astype(int)
    print("\n% w/ non-std carrier", dot20, lasj_round(dt["CarrierNotStd"].mean(), 3))

    # Rename Members to MEMBERS_OR_SUBSCRIBERS
    dt["MEMBERS_OR_SUBSCRIBERS"] = dt["Members"].copy()

    # Create Medical_Orig column based on Sch and Medical
    dt['Medical_Orig']=np.where(dt.Sch=='A',dt.Medical,0)
    
    
    # Create a rule for PBM_Carrier
    rule = "PBM_Carrier"
    notPBM = carrier_exact[carrier_exact["NOT_PBM"] == 1]["Carrier_String"].tolist()
    len(pbmOnlyStr)
    dt["CHANGE"] = np.where(
        (dt["Carrier_String"].isin(notPBM) == False)
        & (dt["Carrier_String"].isin(pbmOnlyStr))
        | (dt["Drugs"] == 1),
        1,
        0,
    )
    dt["CHANGE"].value_counts(dropna=False)

    # Create PBM column based on CHANGE
    dt["PBM"] = dt["CHANGE"].copy()
    print("\nPBMs added - first pbm addition:", y, " ", dt["PBM"].sum())
    
    dt['Rules']=np.where(dt.CHANGE==0,dt['Rules'],dt.Rules+rule + "; ")

    # Update Rules column based on CHANGE and rule
    #dt["Rules"] = dt.apply(
     #   lambda row: row["Rules"] if row["CHANGE"] == 0 else row["Rules"] + rule + "; ",
     #   axis=1,
   # )

    # Create a rule for PBM_Keyword
    rule = "PBM_Keyword"
    strings = ["PBM", "PHARM", "RX", "PRESCRIPT", "DRUG PLAN"]
    notStrings = ["VRX", "NO RX", "MED"]
    dt["CHANGE"] = 0

    # Create a flag for PBM or Carrier_String in alwaysMedStr or Medical
    len(alwaysMedStr)
    dt["NOT"] = np.where(
        (dt["PBM"] == 1)
        | (dt["Carrier_String"].isin(alwaysMedStr))
        | (dt["Medical"] == 1),
        1,
        0,
    )
    dt["NOT"].sum()
    dt["PBM"].sum()

    # Loop over the strings list
    for s in strings:
        # Create a new column FL with 1 if NOT is 0 and either Benefit_Oth or Carrier_Orig contains s, else 0
        dt["FL"] = np.where(
            (dt["NOT"] == 0)
            & ((dt.Benefit_Oth.str.contains(s)) | (dt.Carrier_Orig.str.contains(s))),
            1,
            0,
        )
        # Loop over the notStrings list
        for no in notStrings:
            dt["FL"] = np.where(
                (
                    (dt.Benefit_Oth.str.contains(no))
                    | (dt.Carrier_Orig.str.contains(no))
                ),
                0,
                dt["FL"],
            )
            # Update the FL column with 0 if either Benefit_Oth or Carrier_Orig contains no, else keep the previous value
        # Update the Rules column with the rule and s appended if FL is 0, else keep the previous value
        dt["Rules"] = np.where(dt.FL == 0, dt.Rules, dt.Rules + rule + "_" + s + ";")
        # Update the CHANGE column with 1 if FL is 1, else keep the previous value
        dt["CHANGE"] = np.where(dt.FL == 1, 1, dt["CHANGE"])
    # Update the PBM column with 1 if CHANGE is 1, else keep the previous value
    dt["PBM"] = np.where(dt.CHANGE == 1, 1, dt["PBM"])

    # dt["PBM"] = dt.apply(lambda row: 1 if row["CHANGE"] == 1 else row["PBM"], axis=1)
    # Print some summary statistics
    print(
        "\nEnd",
        rule,
        "  tot PBM added ",
        dt["CHANGE"].sum(),
        " PBM now ",
        dt["PBM"].sum(),
    )

    # Call the countLOBS function with some arguments

    dt = countLOBS(
        dt=dt, fn="Row 220 in " + fn + str(y), mod="", y="", ADD_PLAN_LOBS=True
    )

    # Assign a constant value to the LOBs_Form column
    dt["LOBs_Form"] = dt["Bundled"].copy()

    # Create a new column PMPM_EST with the formula given if Premiums, Members and Duration are positive, else 0
    
    dt["PMPM_EST"] = np.where(
        (dt["Premiums"] > 0) & (dt["Members"] > 0) & (dt["Duration"] > 0),
        lasj_round(
            dt["Premiums"] / (dt["Members"] * (dt["Duration"] / 30.4)), 2
        ),
         0)
    # Update the PMPM_EST column with another formula if Members is less than 1 and Part_EOY is positive, else keep the previous value
   
    dt["PMPM_EST"] = np.where(
        (dt["Members"] < 1) & (dt["Part_EOY"] > 0) ,
        lasj_round(
            dt["Premiums"] / (dt["Part_EOY"] * 1.1 * (dt["Duration"] / 30.4)), 1
        ),
         dt["PMPM_EST"])
    

    rule = "Medical_SchA_RetireeMedKeywords"
    strings = ["MEDICARE", "RETIREE MED"]
    dt["CHANGE"] = 0
    for s in strings:
        # Create a flag for Sch, Medical, LTD, STD, PBM and Benefit_Oth or Carrier_Orig containing s
        dt["FL"] = (
            (dt["Sch"] == "A")
            & (dt["Medical"] == 0)
            & (dt["LTD"] == 0)
            & (dt["STD"] == 0)
            & (dt["PBM"] == 0)
            & (
                (dt["Benefit_Oth"].str.contains(s))
                | (dt["Carrier_Orig"].str.contains(s))
            )
        ).astype(int)

        # Update Rules column based on FL and rule
        dt['Rules']=np.where(dt['FL']==0,dt['Rules'],dt['Rules']+ rule + "_" + s + "; ")
        
        dt["CHANGE"] = np.where((dt["FL"] == 1) | (dt["CHANGE"] == 1), 1, 0)
    # Update CHANGE column based on FL

    dt["CHANGE"].sum()

    # Update Medical column based on CHANGE
    dt["Medical"] = (dt["CHANGE"] == 1) | (dt["Medical"] == 1).astype(int)
    print(
        "\nEnd",
        rule,
        "  tot Medical added b/c Medicare or other retiree med ",
        dt["CHANGE"].sum(),
    )

    # Create a list of new products
    newProducts = [
        "EAP",
        "CI",
        "Hospital",
        "Vol_Life",
        "Add_Travel",
        "FSA",
        "HSA",
        "HRA",
        "LTC",
        "Wellness",
        "Legal",
    ]

   
    # Loop over newProducts
    for p in newProducts:
        rule = p + "_Keywords"
        strings = []
        notStrings = []
        exactKey = [p.upper()]

        # Check the value of p
        if p == "EAP":
            strings = [
                " EAP",
                "BEHAVIOR",
                "MENTAL H",
                "EMPLOYEE AS",
                "EAPFEE",
                "PSYCH",
                "EAP P",
                "EMPLOYEE ASSIST",
                "EAP ADM",
            ]
            notStrings = list(
                set(
                    carrier_exact[carrier_exact["NOT_EAP"] == 1].Carrier_String.tolist()
                    + notWelfCarriers
                    + dentalOnlyStr
                    + visionOnlyStr
                )
            )
        elif p == "CI":
            # Create a list of strings to match
            strings = [
                "CRITICAL IL",
                " CI",
                "CII",
                "CGI",
                "PARALYSIS",
                "CRITICAL CARE",
                "TRANSPLAN",
                "CANCER",
                "GCI",
                "CI VOL",
            ]
            # Create a list of strings to exclude
            notStrings = list(set(notWelfCarriers + dentalOnlyStr + visionOnlyStr))

            # Append another string to the exactKey list
            exactKey.append("C I")

        # Check the value of p
        elif p == "Hospital":
            strings = ["HOSP", "HSP"]
            notStrings = list(set(notWelfCarriers + dentalOnlyStr + visionOnlyStr))

        elif p == "Vol_Life":
            strings = volLifeStrings
            notStrings = list(set(notWelfCarriers + dentalOnlyStr + visionOnlyStr))

        elif p == "LTC":
            strings = ["LONG TERM C", "LONG-TERM C", "LT C", "LTC"]
            notStrings = list(set(notWelfCarriers + dentalOnlyStr + visionOnlyStr))

        elif p == "Add_Travel":
            strings = [
                "BTA",
                "AD&D",
                "TRAVEL",
                "EVACUAT",
                "AD &D",
                "A D AND D",
                "ACCIDENTAL DE",
                "DISMEMB",
                "DEATH AN",
                "ACCIDENT DE",
                "BASIC ADD",
                "VOLUNTARY ADD",
                "ADD LIFE",
                "VADD",
            ]
            notStrings = list(
                set(
                    carrier_exact[carrier_exact["NOT_ADD_TRAVEL"] == 1][
                        "Carrier_String"
                    ].tolist()
                    + dentalOnlyStr
                    + visionOnlyStr
                    + notWelfCarriers
                )
            )
            exactKey = exactKey + ["ADD", "AD&D", "AD"]
        elif p == "Wellness":
            strings = [
                "WELLNE",
                "GYM",
                "FITNESS",
                "KINEMA",
                "EXOS",
                "CAREWISE HE",
                "WELLINT",
                "EB EMPLOYEE",
                "SMOKING CESSATION",
                "TOBACCO CESS",
            ]
            strings.extend(["ACUPU", "CHIROPR", "MASSAGE"])
            notStrings = list(set(dentalOnlyStr + visionOnlyStr + notWelfCarriers))
        elif p == "Legal":
            strings = ["LEGAL"]
            notStrings = list(
                set(alwaysMedStr + lifeDisabOnlyStr + dentalOnlyStr + visionOnlyStr)
            )
        elif p == "FSA":
            strings = [
                "FSA",
                "FLEXIBLE S",
                "FLEXIBLE B",
                "FLEX SPEND",
                "CAFETERIA",
                "SPENDING ACCOUNT",
            ]
        elif p == "HSA":
            strings = ["HSA", "HEALTH SAVINGS"]

        elif p == "HRA":
            strings = ["HRA", "REIMB"]

        if p in ["FSA", "HSA", "HRA"]:
            notStrings = alwaysMedStr

        if len(notStrings) > 0:
            # Create a flag for Carrier_String or Carrier_EIN in notStrings
            dt["NOT"] = np.where(
                (dt["Carrier_String"].isin(notStrings))
                | (dt["Carrier_EIN"].isin(notStrings)),
                1,
                0,
            )

        if p in ["FSA", "HSA", "HRA"]:
            # Update the flag for PMPM_EST greater than pmpm_fi_Medical times 0.3
            dt["NOT"] = np.where(
                (dt["PMPM_EST"] > pmpm_fi_Medical * 0.3) | (dt["NOT"] == 1), 1, 0
            )
        # Create a text for flagging as p based on Benefit_Oth or Carrier_Orig
        Text = "Flag as " + p + " if Benefit_Oth or Carrier_Orig include keywords "

        if p not in dt.columns:
            dt[p] = 0

        dt["CHANGE"] = 0

        for s in exactKey:
            # Update the CHANGE column with 1 if Benefit_Oth is equal to s, else keep the previous value
            dt["CHANGE"] = np.where(dt.Benefit_Oth == s, 1, dt["CHANGE"])
            dt["CHANGE"].value_counts(dropna=False)
            # Update the Rules column with the rule and s appended if CHANGE is 0 or p is 1, else keep the previous value
            dt["Rules"] = np.where(
                (dt.CHANGE == 0) | (dt[p] == 1),
                dt["Rules"] + rule + "_" + s + "; ",
                dt["Rules"],
            )
            # Update the p column with 1 if CHANGE is 1, else keep the previous value
            dt[p] = np.where(dt.CHANGE == 1, 1, dt[p])
        dt["CHANGE"].value_counts(dropna=False)
        dt[dt["NOT"] == 0]

        # Loop over the strings list
        for s in strings:
            # Update the CHANGE column with 1 if NOT is 0 and Benefit_Oth contains s, else keep the previous value

            dt["CHANGE"] = np.where(
                (dt.NOT == 0) & (dt["Benefit_Oth"].str.contains(s)), 1, dt["CHANGE"]
            )
            dt["CHANGE"].value_counts(dropna=False)
            # Check if p is not in a list of values
            if p not in ["Hospital", "Add_Travel", "CI"]:
                # Update the CHANGE column with 1 if NOT is 0 and Carrier_Orig contains s, else keep the previous value
                dt["CHANGE"] = np.where(
                    (dt.NOT == 0) & (dt["Carrier_Orig"].str.contains(s)),
                    1,
                    dt["CHANGE"],
                )

            # Update the Rules column with the rule and s appended if CHANGE is 0 or p is 1, else keep the previous value
            dt["Rules"] = np.where(
                (dt.CHANGE == 0) | (dt[p] == 1),
                dt["Rules"] + rule + "_" + s + "; ",
                dt["Rules"],
            )

            # Update the p column with 1 if CHANGE is 1, else keep the previous value
            dt[p] = np.where(dt.CHANGE == 1, 1, dt[p])
        # Print some summary statistics
        print("\nEnd", rule, Text, " added/total ", dt["CHANGE"].sum())

    #######################
    print(
        "Time taken to get to line 1184:",
        time.time() - start_time_impute_lob,
        "seconds",
    )

    # Wellmark Adjustment - adhoc
    print("\nMed before Wellmark adj ", dt["Medical"].value_counts())
    rule = "Wellmark Med adjustment"

    # Create a flag for Medical, Hospital, Carrier and Premiums/Part_EOY
    dt["FL"] = (
        (dt["Medical"] == 0)
        & (dt["Hospital"] == 1)
        & (dt["Carrier"] == "WELLMARK")
        & ((dt["Premiums"] / dt["Part_EOY"]) > (2 * pmpm_fi_Hospital))
    ).astype(int)
    print("\nCaught in Wellmark Med adjustment", dt["FL"].sum(), "\n")

    # Update Rules column based on FL and rule
    dt['Rules']=np.where(dt['FL']==0,dt['Rules'],dt['Rules']+ rule + "; ")


    # Update Medical column based on FL
    dt["Medical"] = (dt["FL"] == 1) | (dt["Medical"] == 1).astype(int)
    dt = dt.drop(columns=["FL"])
    print("\nMed After Wellmark adj ", dt["Medical"].value_counts())

    rule = "VolLife_ifLife_VolKeyword_"
    Text = "Assign as vol life if Life==1 & Benefit_Oth has Keyword"
    strings = volStringsGeneric
    dt["CHANGE"] = 0
    for s in strings:
        # Create a flag for Vol_Life, Life and Benefit_Oth or Carrier_Orig containing s
        dt["FL"] = (
            (dt["Vol_Life"] == 0)
            & (dt["Life"] == 1)
            & (
                (dt["Benefit_Oth"].str.contains(s))
                | (dt["Carrier_Orig"].str.contains(s))
            )
        ).astype(int)
       # print(s,dt.FL.sum())
        # Update Rules column based on FL and rule
        dt['Rules']=np.where(dt['FL']==0,dt['Rules'],dt['Rules']+ rule + s + "; ")
        # Update CHANGE column based on FL
        dt["CHANGE"] = (dt["FL"] == 1) | (dt["CHANGE"] == 1).astype(int)
    dt['CHANGE'].sum()
    # Create a flag for Life, Vol_Life and Carrier_String in mostlyVol
    dt["FL"] = (
        (dt["Life"] == 1)
        & (dt["Vol_Life"] == 0)
        & (dt["Carrier_String"].isin(mostlyVol))
    ).astype(int)
    dt['FL'].sum()
    
    dt[dt.ACK_ID=='20100611154842P040011142533001'].Carrier_String
    # Update Rules column based on FL and rule
    
    dt['Rules']=np.where(dt['FL']==0,dt['Rules'],dt['Rules']+ rule + dt["Carrier"] + "; ")
    
    # Update CHANGE column based on FL
    dt["CHANGE"] = (dt["FL"] == 1) | (dt["CHANGE"] == 1).astype(int)

    # Update Vol_Life column based on CHANGE
    dt["Vol_Life"] = (dt["CHANGE"] == 1) | (dt["Vol_Life"] == 1).astype(int)
    print(
        "\nEnd ", rule, " ", dt["CHANGE"].sum(), " tot Vol_Life ", dt["Vol_Life"].sum()
    )

    products = ["EAP", "LTC", "Legal", "Add_Travel", "FSA", "Wellness"]
    for p in products:
        rule = "NewLOB_ExclusiveCarrier_" + p
        strings = carrier_exact[carrier_exact[p.upper() + "_ONLY"] == 1][
            "Carrier_String"
        ].unique()
        # Check if p is Wellness
        if p == "Wellness":
            strings = np.concatenate(
                [
                    strings,
                    carrier_exact[carrier_exact["MASSAGE_ONLY"] == 1][
                        "Carrier_String"
                    ].unique(),
                ]
            )
        # Create a new column CHANGE with 1 if p is 0 and Carrier_String is in strings, else 0
       # dt["CHANGE"] = dt.apply(
        #    lambda row: 1 if row[p] == 0 and row["Carrier_String"] in strings else 0,
         #   axis=1,
        #)
        
        dt['CHANGE']=np.where((dt[p]==0) & (dt['Carrier_String'].isin(strings)),1,0)
        # Get the current sum of p column
        now = dt[p].sum()
        # Update the Rules column with the rule appended if CHANGE is 1, else keep the previous value
        
        dt['Rules']=np.where(dt['CHANGE']==1,dt['Rules']+ rule + "; ",
                             dt['Rules'])
        # Update the p column with 1 if CHANGE is 1, else keep the previous value
        #dt[p] = dt.apply(lambda row: 1 if row["CHANGE"] == 1 else row[p], axis=1)
        dt[p]=np.where(dt['CHANGE']==1,1,dt[p])
        # Get the updated sum of p column
        after = dt[p].sum()
        # Print some summary statistics
        print(
            "\nEnd ",
            rule,
            " @335 updated ",
            dt["CHANGE"].sum(),
            " total ",
            p,
            " ",
            after,
            " added ",
            after - now,
        )
    print("dt medical @1314:", dt.Medical.value_counts())

    rule = "Accident_Keywords"
    Text = "Add Accident. Differentiate Accident from AD&D."
    # Update the NOT column with 1 if Carrier_String or Carrier_EIN is in notWelfCarriers, else 0
    dt["NOT"] = np.where(
        (dt["Carrier_String"].isin(notWelfCarriers) == True)
        | (dt["Carrier_EIN"].isin(notWelfCarriers) == True),
        1,
        0,
    )
    dt["NOT"].value_counts(dropna=False)
    # Update the CHANGE column with 1 if Benefit_Oth contains ACCID and Add_Travel is 0 and NOT is 0, else 0
    dt["CHANGE"] = np.where(
        (dt["Benefit_Oth"].str.contains("ACCID"))
        & (dt["Add_Travel"] == 0)
        & (dt["NOT"] == 0),
        1,
        0,
    )
    dt["CHANGE"].value_counts(dropna=False)
    # Create a list of strings to match
    strings = [
        "PERSONAL ACC",
        "ACCIDENT CRIT",
        "ACCIDENT CANC",
        "ACCIDENT HOSP",
        "VOL ACCID",
        "GRPACC",
    ]
    # Loop over the strings list
    for s in strings:
        # Update the CHANGE column with 1 if Benefit_Oth or Carrier_Orig contains s and NOT is 0, else keep the previous value
        dt["CHANGE"] = np.where(
            ((dt.Benefit_Oth.str.contains(s)) | (dt.Carrier_Orig.str.contains(s)))
            & (dt["NOT"] == 0),
            1,
            dt["CHANGE"],
        )

    # Update CHANGE column based on Benefit_Oth
    dt["CHANGE"] = np.where((dt["Benefit_Oth"] == "ACCIDENT"), 1, dt["CHANGE"])

    # Update Rules column based on CHANGE and rule
    dt["Rules"] = np.where(dt["CHANGE"] == 0,dt['Rules'], dt["Rules"] + rule + "; ")
        

    # Check if Accident is a column in dt
    if "Accident" not in dt.columns:
        # Create a column for Accident and set it to zero
        dt["Accident"] = 0

    # Update Accident column based on CHANGE
    dt["Accident"] = np.where((dt["CHANGE"] == 1), 1, dt["Accident"])

    print("\nEnd ", rule, " added/total ", dt["CHANGE"].sum())

    rule = "AD&D_OnlyAccountADDPlan"
    Text = "If Add_Travel_PN and only contract, assume it is AD&D"

    # Drop CT column if exists
    dt = dt.drop(columns=["CT"] if "CT" in dt.columns else [])

    # Create a CT column for the number of Members by State_HQ and ACK_ID
    dt["CT"] = dt.groupby(["State_HQ", "ACK_ID"])["Members"].transform("count")

    # Update CHANGE column based on CT, ADD_TRAVEL_PN, Add_Travel and Accident
    print("dt col at 1368", dt.columns)
    dt["CHANGE"] = (
        (dt["CT"] == 1)
        & (dt["ADD_TRAVEL_PN"] == 1)
        & (dt["Add_Travel"] == 0)
        & (dt["Accident"] == 0)
    ).astype(int)

    # Update Rules column based on CHANGE and rule
    
    dt["Rules"] = np.where(dt["CHANGE"] == 0,dt['Rules'], dt["Rules"] + rule + "; ")
    
    

    # Update Add_Travel column based on CHANGE
    dt["Add_Travel"] = (dt["CHANGE"] == 1) | (dt["Add_Travel"] == 1).astype(int)
    print(
        "\nEnd ", rule, " added ", dt["CHANGE"].sum(), " total ", dt["Add_Travel"].sum()
    )

    # dt.to_csv('n_temp3.csv')

    rule = "AFLAC_ALLSTATE_CIorAccident_ifNoTxt"

    # Update CHANGE column based on Carrier_String, Benefit_Oth and other columns
    dt["CHANGE"] = (
        (dt["Carrier_String"].isin(mostlyVol))
        & (dt["Benefit_Oth"] == "")
        & (dt[["CI", "Accident", "Hospital", "Life", "STD", "LTD"]] == 0).all(axis=1)
    ).astype(int)

    # Update Rules column based on CHANGE and rule
    dt["Rules"] = np.where(dt["CHANGE"] == 0,dt['Rules'], dt["Rules"] + rule + "; ")
    
    
    # Update Accident and CI columns based on CHANGE
    dt = dt.reset_index(drop=True)
    dt.loc[dt["CHANGE"] == 1, "Accident"] = 1
    dt.loc[dt["CHANGE"] == 1, "CI"] = 1
    print("\nEnd ", rule, " added CI and Accident ", dt["CHANGE"].sum())

    rule = "Other_Vol_Keywords"

    # Create a flag for Carrier_String or Carrier_EIN in notWelfCarriers or other columns
    dt["NOT"] = (
        (dt["Carrier_String"].isin(notWelfCarriers))
        | (dt["Carrier_EIN"].isin(notWelfCarriers))
        | (dt[["CI", "Accident", "Hospital", "Vol_Life"]] == 1).any(axis=1)
    ).astype(int)

    # Loop over the volStringsGeneric list
    for s in volStringsGeneric:
        # Create a new column FL with 1 if NOT is 0 and Benefit_Oth contains s, else 0
      #  print(s)
        dt["FL"] = np.where(
            (dt["NOT"] == 0) & (dt["Benefit_Oth"].str.contains(s)), 1, 0
        )
        # Update the FL column with 1 if NOT is 0 and Carrier_Orig contains s, else keep the previous value
        dt["FL"] = np.where(
            (dt["NOT"] == 0) & (dt["Carrier_Orig"].str.contains(s)), 1, dt["FL"]
        )

        # Update the Rules column with the rule and s appended if FL is 0, else keep the previous value
        
        dt['Rules']=np.where(dt.FL==0,dt.Rules,dt.Rules+ rule+ "_"+ s+ "; ")

        dt["CHANGE"] = np.where(dt["FL"] == 1, 1, dt["CHANGE"])
        # Update the CHANGE column with 1 if FL is 1, else keep the previous value

    # Update the CHANGE column with 1 if CI, Accident, Hospital and Vol_Life are all 0 and Benefit_Oth is in a list of values, else keep the previous value
    dt["CHANGE"] = np.where(
        (dt["CI"] == 0)
        & (dt["Accident"] == 0)
        & (dt["Hospital"] == 0)
        & (dt["Vol_Life"] == 0)
        & (dt["Benefit_Oth"].isin(["GAP", "SUPPLEMENTAL", "SUPP", "GAP PLAN"])),
        1,
        dt["CHANGE"],
    )

    # Check if Other_Vol column exists
    if "Other_Vol" not in dt.columns:
        dt["Other_Vol"] = 0

    # Update the Other_Vol column with 1 if CHANGE is 1, else keep the previous value
    
    dt["Other_Vol"] = np.where(dt.CHANGE==1,1,dt['Other_Vol'])
    print(
        "\nEnd ", rule, " added ", dt["CHANGE"].sum(), " total ", dt["Other_Vol"].sum()
    )

    rule = "Other_Vol_AFLACALLSTATE; "
    dt = countLOBS(dt=dt, fn="Row 325 in " + fn)

    # Update the CHANGE column with 1 if Bundled is empty or DummyGroupRecord and Carrier_String is in mostlyVol, else 0
    dt["CHANGE"] = np.where(
        ((dt["Bundled"] == "") | (dt["Bundled"] == "DummyGroupRecord"))
        & (dt["Carrier_String"].isin(mostlyVol) == True),
        1,
        0,
    )

    # Update the Other_Vol column with 1 if CHANGE is 1, else keep the previous value

    dt["Other_Vol"] = np.where(dt["CHANGE"] == 1, 1, dt["Other_Vol"])

    # Update the Rules column with the rule appended if CHANGE is 1, else keep the previous value
    dt["Rules"] = np.where(dt['CHANGE']==1,dt["Rules"] + rule,dt['Rules'])
    
    
    print(
        "\nEnd ", rule, " added ", dt["CHANGE"].sum(), " total ", dt["Other_Vol"].sum()
    )
    print("\nFIRST RECORDED RULE ")

    # Count the LOBS in dt
    dt = countLOBS(dt=dt, fn="Row 320" + fn + "after add new/supplemental")

    rule = "Drop_NA_Carrier_or_NA_Carrier_Relationship"
    start = time.time()
    Text = "Drop if NA Carrier - e.g., investment management - leverage DROP flag from carrier file or NA carrier relatinoship - accounting, audit, investment mgmt,etc."

    print(
        "Time taken to get to line 1503:",
        time.time() - start_time_impute_lob,
        "seconds",
    )
    # Create a list of exact keywords for NA
    naKeywordsExact = [
        "ATTORNEY",
        "ATTORNEYS",
        "EMPLOYEE",
        "TRUSTEE",
        "RECORD KEEPER",
        "RECORDKEEPER",
        "ACCOUNTANT",
        "ACCOUNTING",
        "AUDITOR",
        "ACCOUNTANT AUDITOR",
        "AUDITORS",
        "COMMITTEE MEMBER",
        "FUND EMPLOYEE",
        "INVESTMENT MANAGER",
        "INVESTMENT MANAGEMENT",
        "NONE KNOWN",
        "PLAN EMPLOYEE",
        "PLAN TRUSTEE",
        "PAYROLL AUDITOR",
        "PRINTER",
        "UNION MEMBER",
        "STAFF",
        "STAFF DEVELOPMENT",
        "SOFTWARE PROGRAMMER",
        "EMPLOYEE OF TRUST",
        "INSTRUCTOR",
        "INVESTMENT ADVISOR",
        "INVESTMENT ADVISORY",
        "AUDIT",
        "TAX",
        "SOFTWARE VENDOR",
        "ACTUARY",
        "ACTUARIAL",
        "ACTUARIAL SERVICES",
        "401K PLAN",
        "401K",
        "TAX AND AUDIT",
    ]

    # Create a DROP column based on Carrier_String, notWelfCarriers and LOBS
    dt["DROP"] = (
        (dt["Carrier_String"].isin(notWelfCarriers)) & (dt["LOBS"] == 0)
    ).astype(int)

    # Update DROP column based on LOBs_Form, Carrier, Benefit_Oth and LOBS
    dt["DROP"] = (
        (dt["LOBs_Form"] == "")
        & (~dt["Carrier"].isin([x.upper() for x in majorDental + majorMedical]))
        & (dt["Benefit_Oth"].str.upper().isin(naKeywordsExact))
        & (dt["LOBS"] == 0)
    ) | (dt["DROP"] == 1).astype(int)

    # Drop CT column if exists
    dt = dt.drop(columns=["CT"] if "CT" in dt.columns else [])

    # Create a CT column for the number of Members by State_HQ, EIN and Year
    dt["CT"] = dt.groupby(["State_HQ", "EIN", "Year"])["Members"].transform("count")
    # check here with huge data
    # Create a FL column based on CT, Plan_LOBs and DROP
    dt["FL"] = ((dt["CT"] == 1) & (dt["Plan_LOBs"] != "") & (dt["DROP"] == 1)).astype(
        int
    )

    print("\nFor ", rule, " dont drop if only one & has pn LOB ", dt["FL"].sum())
    # Update the DROP column with 1 if DROP is 1 and FL is 0, else 0

    dt["DROP"] = np.where((dt.DROP == 1) & (dt["FL"] == 0), 1, 0)

    # Call the ruleCapt function with some arguments
    rulesDt = ruleCapt(
        DROP=True,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=75,
    )

    # Filter the dt dataframe by DROP column
    dt = dt[dt["DROP"] == 0]
    print(" and obs remaining after drop @251 ", len(dt))
    # COMPLETED

    rule = "DropNotWelfareTerm"
    Text = (
        "Drop if Benefit_Oth or Carrier_Orig contain a not-ewlfare-term or are a broker"
    )
    dt["DROP"] = 0
    dt["KEY"] = ""
    strings = notWelfareInsTerms + ["BROKER", "WILLIS"]

    # Convert some lists to uppercase
    majors = []
    for l in [majorMedical, majorDental, majorLifeDisab]:
        upper_l = [i.upper() for i in l]
        majors += upper_l

    always = []
    for l in [
        alwaysMedStr,
        dentalOnlyStr,
        visionOnlyStr,
        lifeOnlyStr,
        lifeDisabOnlyStr,
    ]:
        upper_l = [i.upper() for i in l]
        always += upper_l

    print("dt medical @1619:", dt.Medical.value_counts())
    # Create a new column IN with 1 if HSA, HRA, FSA and EAP are all 0 and Bundled is empty or CarrierNotStd is 1 and either Carrier is not in majors or Carrier_String is not in always or PMPM_EST is less than 1, else 0
    dt["IN"] = np.where(
        (dt["HSA"] == 0)
        & (dt["HRA"] == 0)
        & (dt["FSA"] == 0)
        & (dt["EAP"] == 0)
        & ((dt["Bundled"] == "") | (dt["CarrierNotStd"] == 1))
        & (
            (
                (dt["Carrier"].isin(majors) == False)
                & (dt["Carrier_String"].isin(always) == False)
            )
            | (dt["PMPM_EST"] < 1)
        ),
        1,
        0,
    )
    dt["DROP"].value_counts(dropna=False)
    # Loop over the strings list

    for s in strings:
        # Create a new column FL with 1 if DROP is 0 and IN is 1 and Benefit_Oth contains s, else 0
        dt["FL"] = np.where(
            (dt["DROP"] == 0) & (dt["IN"] == 1) & (dt["Benefit_Oth"].str.contains(s)),
            1,
            0,
        )

        # Update the FL column with 1 if DROP is 0 and IN is 1 and Carrier contains s, else keep the previous value
        dt["FL"] = np.where(
            (dt["DROP"] == 0) & (dt["IN"] == 1) & (dt["Carrier"].str.contains(s)),
            1,
            dt["FL"],
        )
        dt["KEY"] = np.where(dt["FL"] == 1, s, dt["KEY"])
        dt["DROP"] = np.where(dt["FL"] == 1, 1, dt["DROP"])

    # Get the brokerNotCar list from the carrier_exact dataframe
    brokerNotCar = carrier_exact[carrier_exact["BROKER_NOT_CARRIER"] == 1][
        "Carrier_String"
    ]
    # Update the DROP column with 1 if Carrier_String or Carrier_EIN is in brokerNotCar, else keep the previous value
    dt["DROP"] = np.where(
        (dt["Carrier_String"].isin(brokerNotCar))
        | (dt["Carrier_EIN"].isin(brokerNotCar)),
        1,
        dt["DROP"],
    )
    dt["DROP"].value_counts(dropna=False)
    # Create a list of exact values to match
    notWelfareExact = ["CONTRACTOR"]
    # Update the DROP column with 1 if Benefit_Oth is in notWelfareExact, else keep the previous value
    dt["DROP"] = np.where(dt["Benefit_Oth"].isin(notWelfareExact), 1, dt["DROP"])

    # Call the ruleCapt function with some arguments
    rulesDt = ruleCapt(
        DROP=True,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=508,
    )

    try:
        # Call the executeDropRuleDT function with some arguments
        dt = executeDropRuleDT(dt=dt, rule=rule)
    except:
        print("Failed to executeDropRuleDT for dt")
    print("dt col 1667", dt.columns)

    rule = "DropServicesOnly"
    Text = "Drop if the carrier only provides services like utilization review, telemedicine, etc "
    strings = carrier_exact[carrier_exact["SERVICES_ONLY"] == 1]["Carrier_String"]

    # Update DROP column based on Carrier_EIN or Carrier_String in strings
    dt["DROP"] = np.where(
        (dt["Carrier_EIN"].isin(strings)) | (dt["Carrier_String"].isin(strings)), 1, 0
    )

    # Loop over servicesNotInsKeywords
    for s in servicesNotInsKeywords:
        # Update DROP column based on Benefit_Oth containing s
        dt["DROP"] = np.where(dt["Benefit_Oth"].str.contains(s), 1, dt["DROP"])

    # Capture the rule in rulesDt
    rulesDt = ruleCapt(
        DROP=True,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=508,
    )
    dt = executeDropRuleDT(dt=dt, rule=rule)

    # dt.to_csv('temp_n4.csv')

    rule = "DropLegal_LawFirm_or_legalServices_NotIns"
    law_firms = [
        "GROOM LAW",
        "LAW GROUP CHART",
        "BROWNSTEIN RA",
        "BALLARD SPAH",
        "BUCHANAN INGERS",
        "COZEN OCONN",
        "KIRSCHNER GAR",
        "JOSEPH REINHAR",
        "ALLOTTA F",
        "EATON PEA",
        "PIERCE ATW",
    ]

    # Drop CT column if exists
    dt = dt.drop(columns=["CT"] if "CT" in dt.columns else [])

    # Create a CT column for LOBS minus Legal
    dt["CT"] = dt["LOBS"] - dt["Legal"]

    # Loop over law firms
    for s in law_firms:
        # Update DROP column based on Carrier_Orig containing s and CT less than one
        dt["DROP"] = np.where(
            (dt["Carrier_Orig"].str.contains(s)) & (dt["CT"] < 1), 1, dt["DROP"]
        )

    # Create a list of not insurance keywords
    notIns = [
        "LEGAL SERVICES",
        "LEGAL COUNSEL",
        "LEGAL SERVICE",
        "FUND LEGAL COUNSEL",
        "ATTORNEY LEGAL",
        "LEGAL FEES",
        "OUTSIDE LEGAL SERVICES",
        "PROVIDES LEGAL SERV",
        "PLAN LEGAL COUNCIL",
        "ATTORNEY LEGAL SERVICES",
        "PREPAID LEGAL PROVIDER",
        "LEGAL CONSULTANT",
        "LEGAL ADVISOR",
    ]

    prob_notLegalIns = ["RELATED PARTY", "INVESTMENT MANAGER", "INVESTMENT PROVIDER"]

    strings = carrier_exact[carrier_exact["LEGAL_ONLY"] == 1]["Carrier_String"].unique()

    # Update DROP column based on CT, Benefit_Oth and CarrierNotStd
    notIns = [i.upper() for i in notIns]
    prob_notLegalIns = [i.upper() for i in prob_notLegalIns]
    dt["DROP"] = np.where((dt["CT"] < 1) & (dt.Benefit_Oth.isin(notIns)), 1,dt.DROP)
    dt["DROP"] = np.where(
        (dt["CT"] < 1)
        & (dt.Benefit_Oth.isin(prob_notLegalIns))
        & (dt.CarrierNotStd == 1),
        1,
        dt.DROP,
    )
    dt["DROP"] = np.where(dt.Carrier_String.isin(strings), 0, dt.DROP)
    dt["DROP"].sum()

    # Capture the rule in rulesDt
    rulesDt = ruleCapt(
        DROP=True,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=535,
    )

    # TODO: commented out
    # Execute the drop rule on dt
    dt = executeDropRuleDT(dt=dt, rule=rule)

    rule = "LOBAssign_Stoploss"
    strings = [
        "STOPLOSS",
        "STOP LOSS",
        "EXCESS",
        "REINS",
        "SPECIFIC",
        "AGGREGATE",
        "Stoploss",
    ]
    dt["CHANGE"] = 0
    for s in strings:
        dt["CHANGE"] = np.where(
            (dt.Stoploss == 0) & (dt.Benefit_Oth.str.contains(s)), 1, dt.CHANGE
        )
        dt["CHANGE"] = np.where(
            (dt.Stoploss == 0) & (dt.Carrier_Orig.str.contains(s)), 1, dt.CHANGE
        )
        # Update CHANGE column based on Stoploss, Benefit_Oth and Carrier_Orig containing s

    # Update Rules column based on CHANGE and rule
    dt["Rules"] = np.where(dt["CHANGE"]==0,dt['Rules'],dt["Rules"] + rule + "_Keyword; ")
    
    

    # Create a flag for Stoploss and Carrier_String in slOnlyStr
    dt["FL"] = ((dt["Stoploss"] == 0) & (dt["Carrier_String"].isin(slOnlyStr))).astype(
        int
    )

    dt["Rules"] = np.where(dt["FL"]==0,dt['Rules'],dt["Rules"] + rule + "_ExclusiveCarrier; ")
    
    # Update Rules column based on FL and rule
   
    # Update CHANGE column based on FL
    dt["CHANGE"] = np.where(dt["FL"] == 1, 1, dt["CHANGE"])

    # Update Stoploss column based on CHANGE
    dt["Stoploss"] = np.where(dt["CHANGE"] == 1, 1, dt["Stoploss"])
    # Capture the rule in rulesDt
    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=551,
    )

    rule = "LOBAdjust_NotMedical_VolorPBMorStoploss"
    Text = "If Tagged as medical but not a medical carrier"
    # Create a Medical_Orig column from Medical
    dt["Medical_Orig"] = dt["Medical"].copy()
    # Drop CT column if exists
    dt = dt.drop(columns=["CT"] if "CT" in dt.columns else [])
    # Create a CT column for the sum of Medical by ACK_ID
    dt["CT"] = dt.groupby("ACK_ID")["Medical"].transform("sum")
    for i in ["STD", "CI", "Stoploss", "PBM"]:
        print(i, dt[i].sum())
    dt["INSCOPE"] = np.where(
        (dt.Medical == 1)
        & (dt.Sch == "A")
        & (dt.CT > 1)
        & (
            (dt.STD == 1)
            | (dt.CI == 1)
            | (dt.Stoploss == 1)
            | (dt.PBM == 1)
            | (dt.Hospital == 1)
            | (dt.Accident == 1)
            | (dt.Other_Vol == 1)
            | (dt.LTC == 1)
            | (dt.EAP == 1)
            | (dt.Wellness == 1)
        ),
        1,
        0,
    )
    dt["INSCOPE"].value_counts(dropna=False)
    sum(dt.CHANGE)
    dt["CHANGE"] = np.where(
        (dt.INSCOPE == 1)
        & (
            (dt.Carrier_String.isin(notMedStrings))
            | (dt.PMPM_EST < 0.25 * pmpm_fi_Medical)
        ),
        1,
        0,
    )

    # Create a new column INSCOPE with 1 if Medical is 1 and Sch is A and CT is greater than 1 and any of STD, CI, Stoploss, PBM, Hospital, Accident, Other_Vol, LTC, EAP or Wellness is 1, else 0
    # Create a new column CHANGE with 1 if INSCOPE is 1 and either Carrier_String is in notMedStrings or PMPM_EST is less than 0.25 times pmpm_fi_Medical, else 0

    dt = executeLobAssign(dt=dt, lob="Medical", NOT=True, ALLOW_NONSTD=True, rule=rule)

    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=563,
    )
    dt.CHANGE.sum()

    rule = "LOBAdjust_NotMedCarrier"
    dt["CHANGE"] = (dt["Medical"] == 1) & (
        dt["Carrier_String"].isin(notMedStrings)
    ).astype(int)

    dt = executeLobAssign(dt=dt, lob="Medical", NOT=True, ALLOW_NONSTD=True, rule=rule)

    rulesDt = ruleCapt(
        sub=dt.copy(), DROP=False, rulesDt=rulesDt, rule=rule, Module=fn, Yr=y, row=75
    )

    rule = "LOBAssign_NotMedicalCarrier_ChangeToVol"
    Text = "If caught by preceding rule (not medical carrier but tagged as medical) assign to voluntary "
    dt["FL"] = (
        (dt["Bundled"] == "Medical")
        & (dt["Medical_Orig"] == 1)
        & (dt["Carrier_String"].isin(notMedStrings))
        & (dt["Sch"] == "A")
        & (dt["LOBS"] == 1)
        & (dt["Medical"] == 0)
        & (dt["PBM"] == 0)
    ).astype(int)
    print("\nCaught init ", rule, " ", dt["FL"].sum())
    dt["CT"] = dt.groupby(["State_HQ", "EIN"])["CI"].transform("sum")
    dt["CHANGE"] = ((dt["FL"] == 1) & (dt["CT"] == 0)).astype(int)

    # Print the update to CI
    print("\nUpdate to CI ", dt["CHANGE"].sum())
    
    # Update Rules column based on CHANGE and rule
    
    dt['Rules']=np.where(dt['CHANGE']==0,dt['Rules'],dt["Rules"] + rule + "CI; ")

    # Update CI column based on CHANGE
    dt["CI"] = (dt["CHANGE"] == 1) | (dt["CI"] == 1).astype(int)

    # dt.to_csv('temp_n5.csv')

    # Create a CT column for the sum of Accident by State_HQ and EIN
    dt["CT"] = dt.groupby(["State_HQ", "EIN"])["Accident"].transform("sum")

    # Update CHANGE column based on FL and CT
    dt["CHANGE"] = ((dt["FL"] == 1) & (dt["CT"] == 0)).astype(int)

    # Print the update to Accident
    print("\nUpdate to Accident ", dt["CHANGE"].sum())

    # Update Rules column based on CHANGE and rule
    dt['Rules']=np.where(dt['CHANGE']==0,dt['Rules'],dt["Rules"] + rule + "Accidnet; ")

    
    # Update Accident column based on CHANGE
    dt["Accident"] = (dt["CHANGE"] == 1) | (dt["Accident"] == 1).astype(int)

    # Create a CT column for the sum of Hospital by State_HQ and EIN
    dt["CT"] = dt.groupby(["State_HQ", "EIN"])["Hospital"].transform("sum")

    # Update CHANGE column based on FL and CT
    dt["CHANGE"] = ((dt["FL"] == 1) & (dt["CT"] == 0)).astype(int)

    # Print the update to Hospital
    print("\nUpdate to Hospital ", dt["CHANGE"].sum())

    # Update Rules column based on CHANGE and rule
    
    dt['Rules']=np.where(dt['CHANGE']==0,dt['Rules'],dt["Rules"] + rule + "Hospital; ")


    # Update CHANGE column based on FL, Hospital, Accident and CI
    dt["CHANGE"] = (
        (dt["FL"] == 1) & (dt[["Hospital", "Accident", "CI"]] == 0).all(axis=1)
    ).astype(int)

    # Print the update to Other Vol
    print("\nOther ", dt["CHANGE"].sum())
    dt['Rules']=np.where(dt['CHANGE']==0,dt['Rules'],dt["Rules"] + rule + "Other_Vol; ")

    # Update Rules column based on CHANGE and rule
    

    # Update Other_Vol column based on CHANGE
    dt["Other_Vol"] = (dt["CHANGE"] == 1) | (dt["Other_Vol"] == 1).astype(int)

    # Update CHANGE column based on FL
    dt["CHANGE"] = (dt["FL"] == 1).astype(int)

    # Capture the rule in rulesDt
    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=594,
    )


    rule = "DropObviousDuplicates"
    Text = (
        "Drop obvious duplicates - EINPN, members, premiums, carrier, Sch all the same"
    )
    # Create a new column KEY by concatenating some columns
    # dt=dt.sort_values(by=['ACK_ID','Members','Premiums','Carrier','Sch','Medical',
    #                      'Life','Dental'],kind='stable')

    
    dt['KEY']=dt['ACK_ID']+dt['Members'].astype(str)+dt['Premiums'].astype(str)+dt['Carrier'].astype(str)+dt['Sch'].astype(str)+dt['Medical'].astype(str)+dt['Life'].astype(str)+dt['Dental'].astype(str)
    
    

    # key=['ACK_ID','Members','Premiums','Carrier','Sch','Medical','Life','Dental']
    # Sort the dt dataframe by KEY and Premiums in descending order
    dt = dt.sort_values(["KEY", "Premiums",'Carrier_EIN','Carrier_Orig'], ascending=[True, False,True,True])
    # Create a new column IND with the rank of each row within each group of KEY
    dt["IND"] = dt.groupby("KEY")["Members"].rank(method="first")
    # Create a new column DROP with 1 if IND is greater than 1 and Sch is A, else 0
    
    
    dt['DROP']=np.where( (dt['IND']>1) & (dt['Sch']=='A'),1,0)
    
    # Print the sum of DROP column
    print(dt["DROP"].sum())

    # Call the ruleCapt function with some arguments
    rulesDt = ruleCapt(
        DROP=True,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=600,
    )
    # TODO commented out
    # Call the executeDropRuleDT function with some arguments
    dt = executeDropRuleDT(dt=dt, rule=rule)

    print(
        "Time taken to get to line 2010:",
        time.time() - start_time_impute_lob,
        "seconds",
    )

    # dt.to_csv('temp_n5n5.csv')

    rule = "Drop_UnreasonableDuration_with_NoLOB_noMembers_or_noPremiums"
    Text = "Drop if duration is still less than 40 days or > 1000 days and not only contract. If duration is < 5, drop regardless"
    # Create a new column DROP with 1 if Duration is less than 50 or greater than 1000 and any of LOBS, Premiums, Sch and Members or PART_MAX is 0, else 0
    dt["DROP"] = np.where(
        (
            ((dt.Duration < 50) | (dt.Duration > 1000))
            & (
                (dt.LOBS == 0)
                | (dt["Premiums"] == 0)
                | ((dt["Sch"] == "A") & (dt["Members"] == 0))
                | (dt["PART_MAX"] == 0)
            )
        ),
        1,
        0,
    )

    dt["DROP"].sum()

    # Call the ruleCapt function with some arguments
    rulesDt = ruleCapt(
        DROP=True,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=613,
    )

    dt = executeDropRuleDT(dt=dt, rule=rule)

    rule = "Drop_SchC_Marginal"
    Text = (
        "Drop Sch C obs if no CarrierRel write-in, carrier couldn'tbe found and premiums < 15K"
        + ", no LOBS (and group has other contracts per most drop rules)"
    )

    dt["FL"] = (dt["CarrierNotStd"] == 0) & (dt["Premiums"] > 15000).astype(int)
    dt["CT"] = dt.groupby("ACK_ID")["FL"].transform("sum")
    dt["DROP"] = (
        (dt["Sch"] == "C")
        & (dt["CT"] > 1)
        & (dt["CarrierNotStd"] == 1)
        & (dt["Premiums"] < 15000)
        & (dt["Benefit_Oth"] == "")
        & (dt["LOBS"] == 0)
    ).astype(int)
    print(dt["DROP"].sum())

    rulesDt = ruleCapt(
        DROP=True,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=625,
    )
    dt = executeDropRuleDT(dt=dt, rule=rule)

    # olddt=dt.copy()

    print("dt medical @2067:", dt.Medical.value_counts())
    # NOTE: START HERE< medical looks good
    rule = "LOBAssign_SchC"
    Text = "For sch C assign as Medical, Dental, Vision, STD "
    products = ["Medical", "Dental", "Vision", "STD", "LTD", "Life"]

    # dt.to_csv('temp_n6.csv')

    for p in products:
        print("\n\n")
        print(p, " - start SchedC LOB assignment for ", y, dot20)
        dt["INSCOPE"] = np.where(
            ((dt["Sch"] == "C") | (dt["LOBS"] == 0))
            & ((dt["CarrierNotStd"] == 0) | (dt["Premiums"] > 5000))
            & (dt["PBM"] == 0),
            1,
            0,
        )
        dt.INSCOPE.value_counts(dropna=False)

        dt["PRODCAR"] = 0
        te = dt[(dt["INSCOPE"] == 1) & (dt[p] == 1)]
        if len(te) > 0:
            raise Exception("\nAlready ", len(te), " SchC ", p, " accounts ")
        if p in ["STD", "LTD", "Life"]:
            dt["INSCOPE"] = np.where(
                (dt["Accident"] == 1)
                | (dt["Hospital"] == 1)
                | (dt["Vol_Life"] == 1)
                | (dt["Life"] == 1)
                | (dt["STD"] == 1)
                | (dt["LTD"] == 1),
                0,
                dt["INSCOPE"],
            )
        # First, check for keywords or for always/only LOB
        pmpmLim = None
        notStrings = []
        strings = []
        membToEmp = None
        isProduct = []
        if p == "Medical":
            isProduct = alwaysMedStr
            membToEmp = 1.35
            pmpmLim = 5
            strings = ["MEDICAL", "MEDICARE", "RETIREE HEALTH", "RETIREE MEDICAL"]
            notStrings = [
                "TELEMED",
                "MEDICAL CONSULT",
                "MEDICAL GAP",
                "LIMITED MED",
                "MEDICAL ABR",
                "MEDICAL LEAV",
                "REINSUR",
                "COUNTRY MED",
                "EXPAT",
                "GLOBAL",
            ]

        elif p == "Dental":
            isProduct = dentalOnlyStr
            membToEmp = 1.35
            pmpmLim = 1
            strings = ["DENTAL"]
            notStrings = ["ACCIDENT", "OCCIDENTA"]

        elif p == "Vision":
            isProduct = visionOnlyStr
            membToEmp = 1
            pmpmLim = 0.4
            strings = ["VISION"]
            notStrings = ["ENVISION", "UNRELATED"]

        elif p == "LTD":
            membToEmp = 1
            pmpmLim = 1
            strings = ["LTD", "LONG T", "DBL"]
            notStrings = ["UNRELATED", "MED", "SHORT"]

        elif p == "STD":
            isProduct = disabOnlyStr + lifeDisabOnlyStr
            membToEmp = 1
            pmpmLim = 2
            strings = [
                "STD",
                "SHORT TERM D",
                "TEMPORARY DI",
                "SHORT-TERM D",
                "STATUTORY DI",
                "STATE DISAB",
                "WEEKLY IN",
                "NJ DISAB",
                "NY DISAB",
                "TDB",
            ]
            notStrings = ["UNRELATED"]

        elif p == "Life":
            isProduct = lifeOnlyStr
            membToEmp = 1
            pmpmLim = 2
            strings = ["LIFE"]

        # Create a new column MEMBERS_EST with the lasj_rounded value of membToEmp times Part_EOY if Members is less than 1, else Members

        dt["MEMBERS_EST"] = np.where(
            dt["Members"] < 1, lasj_round(membToEmp * dt["Part_EOY"]), dt.Members
        )

        # Check if isProduct list is not empty
        if len(isProduct) > 0:
            # Create a new column PRODCAR with 1 if INSCOPE is 1 and Carrier_String is in isProduct, else 0
            dt["PRODCAR"] = np.where(
                (dt.INSCOPE == 1) & (dt.Carrier_String.isin(isProduct)), 1, 0
            )

        # Print some summary statistics
        print(
            "\nSchC obs caught as ",
            p,
            " bc Carrier is always ",
            p,
            " ",
            dt["PRODCAR"].sum(),
        )

        dt["KEYWD"] = 0
        for s in strings:
            # Update the KEYWD column with 1 if INSCOPE is 1 and Benefit_Oth contains s, else keep the previous value
            dt["KEYWD"] = np.where(
                (dt["INSCOPE"] == 1) & (dt["Benefit_Oth"].str.contains(s)),
                1,
                dt["KEYWD"],
            )
        dt["KEYWD"].sum()

        if p == "Medical":
            # Update the KEYWD column with 1 if INSCOPE is 1 and Benefit_Oth is EPO, else keep the previous value
            dt["KEYWD"] = np.where(
                (dt["INSCOPE"] == 1) & (dt["Benefit_Oth"] == "EPO"), 1, dt["KEYWD"]
            )

        if p == "STD":
            # Update the KEYWD column with 1 if INSCOPE is 1 and LTD is 0 and Benefit_Oth contains DISAB, else keep the previous value
            dt["KEYWD"] = np.where(
                (dt["INSCOPE"] == 1)
                & (dt["LTD"] == 0)
                & (dt["Benefit_Oth"].str.contains("DISAB")),
                1,
                dt["KEYWD"],
            )

        # Check if notStrings list has at least one element
        if len(notStrings) >= 1:
            print(
                " and those caught with keywords b4 adjust for notStrings ",
                dt["KEYWD"].sum(),
            )
            # Loop over the notStrings list
            for s in notStrings:
                dt["KEYWD"] = np.where(
                    (dt["INSCOPE"] == 1) & (dt["Benefit_Oth"].str.contains(s)),
                    0,
                    dt["KEYWD"],
                )
                # Update the KEYWD column with 0 if INSCOPE is 1 and Benefit_Oth contains s, else keep the previous value

        print(" and separately those found with keywords ", dt["KEYWD"].sum())
        dt["Rules"] = np.where(
            (dt["INSCOPE"] == 1) & (dt[p] == 0) & (dt["KEYWD"] == 1),
            str(dt["Rules"]) + " LOBAssign_" + str(p) + "_Keyword",
            dt["Rules"],
        )
        dt[p] = np.where(dt["KEYWD"] == 1, 1, dt[p])
        dt["Rules"] = np.where(
            (dt["INSCOPE"] == 1) & (dt[p] == 0) & (dt["PRODCAR"] == 1),
            str(dt["Rules"]) + " LOBAssign_" + str(p) + "_ExclusiveCar",
            dt["Rules"],
        )
        dt[p] = np.where(dt["PRODCAR"] == 1, 1, dt[p])

        dt[p].sum()

        dt["OthMemb"] = np.where((dt[p] == 1), dt["MEMBERS_EST"], 0)
        dt["OthMemb"] = dt.groupby("ACK_ID")["OthMemb"].transform("sum")
        dt["HASPLAN"] = (dt[p.upper() + "_PN"] == 1).astype(int)

        dt["ExpMemb"] = np.where((dt["HASPLAN"] == 1), dt["MEMBERS_EST"], 0)

        dt["ResidCMemb"] = np.where(
            (dt["INSCOPE"] == 1) | (dt["HASPLAN"] == 0),
            0,
            np.where(
                lasj_round(dt["ExpMemb"] - dt["OthMemb"]) < 0,
                0,
                lasj_round(dt["ExpMemb"] - dt["OthMemb"]),
            ),
        )

        # NOTE: changed this to add a p.upper condition because wasnt correct
        #  print("carrier_exact col at line 2270:", carrier_exact.columns)
        # print("dt col at line 2270:", dt.columns.tolist())
        if p == "Medical":
            notProductCarString = notMedStrings
        else:
            notProductCarString = (
                carrier_exact.loc[
                    carrier_exact["NOT_" + p.upper()] == 1, "Carrier_String"
                ].tolist()
                + alwaysMedStr
            )
        dt["NOT"] = (
            dt["Carrier_String"].isin(notWelfCarriers)
            | dt["Carrier_String"].isin(notProductCarString)
        ).astype(int)

        # Create a ADDPROD column based on ResidCMemb, NOT, HASPLAN, INSCOPE and other columns
        dt["ADDPROD"] = (
            (dt["ResidCMemb"] > 150)
            & (dt["NOT"] == 0)
            & (dt["HASPLAN"] == 1)
            & (dt["INSCOPE"] == 1)
            & (
                dt[
                    [
                        "FSA",
                        "HRA",
                        "HSA",
                        "PMPM_EST",
                        "PBM",
                        "CarrierNotStd",
                        "CI",
                        "EAP",
                    ]
                ]
                == 0
            ).all(axis=1)
            & (dt[p] == 0)
        ).astype(int)

        # Update Rules column based on ADDPROD and p
        
        dt['Rules']=np.where(dt["ADDPROD"] == 0,dt['Rules'],
                             dt["Rules"] + " LOBAssign_" + p + "_ExclusiveCar")

        # Update p column based on ADDPROD
        dt[p] = (dt["ADDPROD"] == 1) | (dt[p] == 1).astype(int)
        print("\nAdditional ADDPROD tagged as ", p, " .", dt["ADDPROD"].sum())

        # Create a CT column based on INSCOPE and p
        dt["CT"] = ((dt["INSCOPE"] == 1) & (dt[p] == 1)).astype(int)

        # Sum the CT column by ACK_ID
        dt["CT"] = dt.groupby("ACK_ID")["CT"].transform("sum")

        # Print the table of CT values
        print(dt["CT"].value_counts())

        # Create a SchCOther column based on ASOSchC and InsSchC
        dt["SchCOther"] = ((dt["ASOSchC"] == 0) & (dt["InsSchC"] == 0)).astype(int)

        # Create a FL column based on PRODCAR
        dt["FL"] = (dt["PRODCAR"] == 1).astype(int)

        # Update FL column based on INSCOPE, KEYWD, p, CT, SchCOther, PMPM_EST and Sch
        dt["FL"] = (
            (dt["INSCOPE"] == 1)
            & (dt["KEYWD"] == 0)
            & (dt["FL"] == 0)
            & (dt[p] == 1)
            & (dt["CT"] > 1)
            & (
                (dt["SchCOther"] == 1)
                | ((dt["CT"] > 1) & (dt["PMPM_EST"] < pmpmLim))
                | (dt["Sch"] == "A")
            )
        ).astype(int)

        # Create a CT2 column for the sum of FL by ACK_ID
        dt["CT2"] = dt.groupby("ACK_ID")["FL"].transform("sum")
        dt["FL"] = np.where(dt["CT2"] == dt["CT"], 0, dt["FL"])
        print(
            "\nRevert assignment if many accounts per EIN tagged as ",
            p,
            " if one potentially not a fit ",
            dt["FL"].sum(),
        )
        if p in dt.columns:
            dt[p] = np.where(dt["FL"] == 1, 0, dt[p])

        te = dt[(dt["INSCOPE"] == 1) & (dt[p] == 1) & (dt["Sch"] == "C")]
        print("\nSchC obs tagged as ", p, " this far ", len(te))

        te = dt[(dt["INSCOPE"] == 1) & (dt[p] == 1) & (dt["Sch"] == "A")]
        print(" and no-LOB SchA obs tagged as ", p, " ", len(te), dot20end)

    

    dt = deDup_SameCarMembPremsBundle(dt=dt, y=y, fn=fn)
    rulesDt = ruleCapt(
        DROP=True,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule="deDup_sameCarMembPremsBundle_alreadydropped",
        Module=fn,
        Yr=y,
        Description="",
        row=739,
    )
    # NOTE: STOP, by now Medical has been messed up
    print("dt medical @2380:", dt.Medical.value_counts())
    # TEST AND QA AND SAVE INTERIM VERSION
    # Create a list of variables to drop
    dropVars = list(
        set(["ResidCMemb", "ADDPROD", "SchCOther", "PRODCAR", "Unlikely"] + junkVars)
    )
    dropVars = list(set(dropVars).intersection(set(dt.columns)))
    dt = dt.drop(columns=dropVars)
    dt = dt.sort_values(by="ACK_ID")

    # Create a file name for saving
    midFi = wd_delme + "midimputeLob" + str(y) + ".pkl"
    dt.to_pickle(midFi)
    print("\nInterim saved as ", midFi)
    dt.shape
    # #pd.read_pickle(midFi).to_csv('midme.csv')

    # Set a flag for loading
    LOAD_MID = False

    # Check if LOAD_MID is True
    if LOAD_MID:
        # JUST FOR QA/TESTING
        midFi = wd_delme + "midimputeLob" + str(y) + ".pkl"
        # Load the dt dataframe from the pkl file
        dt = pd.read_pickle(midFi)
        info = dt.copy()
        print("\n", midFi, "saved at ")
        print(info["mtime"])

    #
    rule = "LOBAssign_TPA"
    Text = "TPA if keywords or if TPA-only carrier."
    tpaOnly = carrier_exact[carrier_exact["TPA_ONLY"] == 1]["Carrier_String"].tolist()
    fiOnly = carrier_exact[carrier_exact["FI_ONLY"] == 1]["Carrier_String"].tolist()
    strings = [
        "THIRD PAR",
        "3RD PART",
        "TPA",
        "PARTY ADMIN",
        "CONTRACT AD",
        "CLAIMS PR",
        "CLAIMS AD",
        "CLAIM AD",
        "CLAIM PR",
    ]
    dt["FL"] = 0
    dt["IN"] = np.where(
        (dt["EAP"] == 0) & (dt["Life"] == 0) & (~dt["Carrier_String"].isin(fiOnly)),
        1,
        0,
    )
    dt["FL"].value_counts(dropna=False)
    for s in strings:
        dt["FL"] = np.where(
            (dt["IN"] == 1) & (dt["Carrier_Orig"].str.contains(s)),
            1,
            dt["FL"],
        )
        dt["FL"] = np.where(
            (dt["IN"] == 1) & (dt["Benefit_Oth"].str.contains(s)),
            1,
            dt["FL"],
        )

    dt["FL"] = np.where(
        dt["Benefit_Oth"].isin(
            ["TPA", "THIRD PARTY ADMINISTRATOR", "THIRD PARTY ADMIN"]
        ),
        1,
        dt["FL"],
    )

    dt["Rules"] = np.where(
        dt["FL"] == 0, dt["Rules"], dt["Rules"] + rule + "_Keyword; "
    )
    dt["CHANGE"] = np.where(dt["FL"] == 1, 1, 0)
    dt["CHANGE"].value_counts(dropna=False)

    dt["FL"] = np.where(
        (dt["CHANGE"] == 0) & (dt["Carrier_String"].isin(tpaOnly) == True), 1, 0
    )

    dt["FL"].value_counts(dropna=False)
    dt["Rules"] = np.where(
        dt["FL"] == 0, dt["Rules"], dt["Rules"] + rule + "_Carrier; "
    )
    dt["CHANGE"] = np.where(dt["FL"] == 1, 1, dt["CHANGE"])

    if not ("TPA" in dt.columns):
        dt["TPA"] = 0

    dt["TPA"] = (dt["CHANGE"] == 1).astype(int)
    print("\nEnd ", rule, " TPAs ", dt["TPA"].sum())
    dt.shape

    rule = "LOBAssign_Med_CarrierAlwaysMed; "
    Text = "Medical if carrier is always medical"
    dt["IN"] = np.where(
        ((dt["Sch"] == "A") & (dt["PMPM_EST"] > 0.7 * pmpm_fi_Medical))
        | ((dt["Sch"] == "C") & (dt["PMPM_EST"] > 0.7 * pmpm_aso_Medical)),
        1,
        0,
    )

    # Update CHANGE column based on Medical, Carrier_String, MEDICAL_PN, Plan_LOBs and IN
    dt["CHANGE"] = np.where(
        (dt["Medical"] == 0)
        & (dt["Carrier_String"].isin(alwaysMedStr))
        & (~dt["Carrier_String"].isin(notMedStrings))
        & ((dt["MEDICAL_PN"] == 1) | (dt["Plan_LOBs"] == "") | (dt["IN"] == 1)),
        1,
        0,
    )

    print("\n\nCaught @730 initially by", rule, " ", dt["CHANGE"].sum())

    # Create a FL column for the sum of Medical by State_HQ, EIN, PN and Year
    dt["FL"] = dt.groupby(["State_HQ", "EIN", "PN", "Year"])["Medical"].transform("sum")

    # Update FL column based on FL, IN, CHANGE and Sch
    print("line 2489, FL col in", type(dt["FL"]))
    dt["FL"] = (
        (dt["FL"] > 0) & (dt["IN"] == 0) & (dt["CHANGE"] == 1) & (dt["Sch"] == "A")
    ).astype(int)
    print(" Reverted b/c not only Medical & PMPMs weird ", dt["FL"].sum())

    # Update CHANGE column based on FL
    dt["CHANGE"] = np.where(dt.FL == 1, 0, dt.CHANGE)
    print("\nCaught by", rule, "initially ", dt["CHANGE"].sum())

    # Execute the lob assign on dt for Medical
    dt = executeLobAssign(dt=dt, lob="Medical", rule=rule)

    # Capture the rule in rulesDt
    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=788,
    )
    # TESTING/UPDATE VARS
    checkForBads(dtCheck=dt.copy(), txt="@680 after" + rule, y=y)

    # Count the LOBS in dt

    dt = countLOBS(dt=dt, y=y)
    len(dt)
    dt["Accident"] = np.where(dt.Accident == True, 1, 0)
    dt["Other_Vol"] = np.where(dt.Other_Vol == True, 1, 0)

    # Create a DurationMths column from Duration
    dt["DurationMths"] = lasj_round(dt["Duration"] * 0.03296, 1)
    # Print the table of DurationMths values
    print(dt["DurationMths"].value_counts())
    
  #  olddt=dt.copy()
 # dt=olddt.copy()
  

    # SMOOTH DATA OUTLIERS, ADD PMPM ASSUMPTIONS TO EXECUTE
    # Apply the pmpm threshold on dt
    # NOTE: added try/except. delete later
    dt = apply_pmpm_thresh(dt=dt, mod=fn)
    # dt.to_csv('highcheck.csv')
    dt = smoothExtremeDataErrors(dt=dt, y=y, EXEMPT=True, mod=fn)

    # done completed
    # Capture the rule in rulesDt
    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule="smoothExtremeDAtaErrors_alreadyDropped",
        Module=fn,
        Yr=y,
        Description="",
        row=793,
    )

    rule = "Stoploss_NotMedical"
    Text = "If SchA and intially medical but not-med-carrier and PMPM > 25 and < 210 and not a dental/vision/pbm-only carrier and not drugs and  < 2 other LOBs assume it is SL. If its not an SL carrier, just mark not med"

    notStoploss = carrier_exact[carrier_exact["NOT_STOPLOSS"] == 1][
        "Carrier_String"
    ].tolist()

    notStoploss = list(
        set(
            notStoploss + dentalOnlyStr + visionOnlyStr + lifeDisabOnlyStr + lifeOnlyStr
        )
    )

    # Create a new column CHANGE with 1 if Medical_Orig is 1 and Sch is A and Medical is 0 and FI_RATIO is greater than 2 and PMPM_EST is between 25 and 210 and Bundled is empty or DummyGroupRecord and HRA, FSA, HSA, PBM and Stoploss are all 0 and Carrier_String is not in notStoploss, else 0
    dt["CHANGE"] = np.where(
        (dt["Medical_Orig"] == 1)
        & (dt["Sch"] == "A")
        & (dt["Medical"] == 0)
        & (dt["FI_RATIO"] > 2)
        & (dt["PMPM_EST"] > 25)
        & (dt["PMPM_EST"] < 210)
        & ((dt["Bundled"] == "") | (dt["Bundled"] == "DummyGroupRecord"))
        & (dt["HRA"] == 0)
        & (dt["FSA"] == 0)
        & (dt["HSA"] == 0)
        & (dt["PBM"] == 0)
        & (dt["Stoploss"] == 0)
        & (dt["Carrier_String"].isin(notStoploss) == False),
        1,
        0,
    )

    # Call the executeLobAssign function with some arguments
    dt = executeLobAssign(dt=dt, lob="Stoploss", rule=rule)
    # Call the ruleCapt function with some arguments
    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=810,
    )
    # Create a new column INSCOPE with 1 if Sch is A and PBM and Stoploss are 0 and MEMBERS_OR_SUBSCRIBERS is less than 3, else 0
    dt["INSCOPE"] = np.where(
        (dt["Sch"] == "A")
        & (dt["PBM"] == 0)
        & (dt["Stoploss"] == 0)
        & (dt["MEMBERS_OR_SUBSCRIBERS"] < 3),
        1,
        0,
    )

    # Print some summary statistics
    print("\n\nStart three Sched A drop rules total in-scope ", dt["INSCOPE"].sum())

    
   
    
    rule = "DropMissingMembAndPremsPartorLOB"
    print("\nStart ", rule, "obs now ", len(dt))
    Text = "Sch A:Drop if Members < 3 and Premiums < 100 or no LOBS or Part_EOY==0 and several contracts"

    # Create a new column DROP with 1 if INSCOPE is 1 and Bundled is empty and Premiums is less than 100, else 0
    dt["DROP"] = np.where(
        (dt["INSCOPE"] == 1) & (dt["Bundled"] == "") & (dt["Premiums"] < 100), 1, 0
    )

    # Update the DROP column with 1 if INSCOPE is 1 and Bundled is empty and either Premiums is less than 100 or Part_EOY is less than 3, else keep the previous value
    dt["DROP"] = np.where(
        (dt["INSCOPE"] == 1)
        & (dt["Bundled"] == "")
        & ((dt["Premiums"] < 100) | (dt["Part_EOY"] < 3)),
        1,
        dt["DROP"],
    )

    # Update the DROP column with 1 if INSCOPE is 1 and Part_EOY is less than 3 and Premiums is less than 100, else keep the previous value
    dt["DROP"] = np.where(
        (dt["INSCOPE"] == 1) & (dt["Part_EOY"] < 3) & (dt["Premiums"] < 100),
        1,
        dt["DROP"],
    )
    

    # Drop the CT column from the dt dataframe
    if "CT" in dt.columns:
        dt = dt.drop(columns="CT")
    # Create a new column CT with the sum of 1 minus DROP within each group of State_HQ, EIN and Year
    dt["CT"] = dt.groupby(["State_HQ", "EIN", "Year"])["DROP"].transform(
        lambda x: (1 - x).sum()
    )
    # Update the DROP column with 1 if DROP is 1 and CT is greater than 0, else keep the previous value
    dt["DROP"] = np.where((dt["DROP"] == 1) & dt["CT"] > 0, 1, dt["DROP"])
    dt['DROP'].sum()
    
    rulesDt = ruleCapt(
        DROP=True,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=819,
    )
    dt = executeDropRuleDT(dt=dt, rule=rule)

    # TESTING/CLEANUP JUNK
    dt = cleanupJunk(dt=dt)
    if "PMPM" in dt.columns:
        raise Exception("\nPMPM is found in dt ")
    else:
        print("\nNo PMPM @526")

    rule = "NotMedical_StoplossOnly"
    pmpmLim = lasj_round(0.45 * pmpm_fi_Medical)
    Text = (
        "Sch A - Not medical if stoploss and PMPM > 0 & < "
        + str(pmpmLim)
        + " and has other medical contracts"
    )
    dt["CT"] = dt.groupby("ACK_ID")["Medical"].transform("sum")
    dt["CHANGE"] = np.where(
        (dt["Sch"] == "A")
        & (dt["Stoploss"] == 1)
        & (dt["Medical"] == 1)
        & (dt["CT"] > 1)
        & (dt["PMPM_EST"] < pmpmLim)
        & (dt["PMPM_EST"] > 1)
        & (~dt["Carrier_String"].isin(alwaysMedStr)),
        1,
        0,
    )

    dt = executeLobAssign(dt=dt, lob="Medical", NOT=True, ALLOW_NONSTD=True, rule=rule)
    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description=Text,
        row=837,
    )

    rule = "HMOAdjust_Keywords"
    strings = ["HMO", "HEALTH MAINT"]
    dt["CHANGE"] = 0
    for s in strings:
        # Update CHANGE column based on HMO, Benefit_Oth and Carrier_Orig containing s
        dt["CHANGE"] = np.where(
            (dt["HMO"] == 0)
            & (
                (dt["Benefit_Oth"].str.contains(s))
                | (dt["Carrier_Orig"].str.contains(s))
            ),
            1,
            dt["CHANGE"],
        )
    # Update CHANGE column based on HMO and Benefit_Oth in EPO or EPO CONTRACT
    dt["CHANGE"].sum()

    dt["CHANGE"] = np.where(
        (dt["HMO"] == 0) & (dt["Benefit_Oth"].isin(["EPO", "EPO CONTRACT"])),
        1,
        dt["CHANGE"],
    )

    # Update Rules column based on CHANGE and rule
    dt["Rules"] = dt.apply(
        lambda row: row["Rules"] if row["CHANGE"] == 0 else row["Rules"] + rule + " ",
        axis=1,
    )

    # Update HMO column based on CHANGE
    dt["HMO"] = np.where(dt["CHANGE"] == 1, 1, dt["HMO"])
    print("\nEnd", rule, " ", dt["CHANGE"].sum(), " # HMO ", dt["HMO"].sum())

    # Create a rule for HMOAdjust_ExclusiveCarrier
    rule = "HMOAdjust_ExclusiveCarrier"
    hmoOnlyStrings = list(
        carrier_exact[carrier_exact["HMO_ONLY"] == 1]["Carrier_String"].unique()
    )
    dt["CHANGE"] = (
        (dt["HMO"] == 0) & (dt["Carrier_String"].isin(hmoOnlyStrings))
    ).astype(int)
    
    dt["Rules"] = np.where(dt['CHANGE']==0,dt['Rules'],dt['Rules']+rule+ " ")
    
    dt["HMO"] = (dt["CHANGE"] == 1) | (dt["HMO"] == 1).astype(int)

    # Update PPO column based on Carrier_String in hmoOnlyStrings
    dt["PPO"] = (
        (~dt["Carrier_String"].isin(hmoOnlyStrings)) & (dt["PPO"] == 1)
    ).astype(int)
    print("\nEnd ", rule, " ", dt["CHANGE"].sum(), " tot HMO ", dt["HMO"].sum())

    # Create a rule for PPOAdjust_Keywords
    rule = "PPOAdjust_Keywords"
    strings = ["PPO", "PREFERRED PROVID"]
    dt["CHANGE"] = 0
    for s in strings:
        # Update CHANGE column based on PPO, HMO, Benefit_Oth and Carrier_Orig containing s
        dt["CHANGE"] = (
            (dt["PPO"] == 0)
            & (dt["HMO"] == 0)
            & (
                (dt["Benefit_Oth"].str.contains(s))
                | (dt["Carrier_Orig"].str.contains(s))
            )
        ) | (dt["CHANGE"] == 1).astype(int)

    # Update Rules column based on CHANGE and rule
   
    dt["Rules"] = np.where(dt['CHANGE']==0,dt['Rules'],dt['Rules']+rule+ " ")

  
    # Update PPO column based on CHANGE
    dt["PPO"] = (dt["CHANGE"] == 1) | (dt["PPO"] == 1).astype(int)
    print("\nEnd ", rule, y, " ", dt["CHANGE"].sum(), " # PPO ", dt["PPO"].sum())

    # ROUTINE TESTING/UPDATE DERIVED VARS
    # Count the LOBS in dt
    dt = countLOBS(dt=dt, fn="@770" + fn, y=y)

    # Call the printFITotals function with some arguments
    #  printFITotals(dt=dt, y=y, mod="@770 in " + fn, EXEMPT=True)

    print(
        "Time taken to get to line 2750:",
        time.time() - start_time_impute_lob,
        "seconds",
    )

    rule = "LOBRevert_NOTGroupTermLife_BC_VolLife; "
    doubleLim = lasj_round(pmpm_fi_Life * 0.85 + pmpm_fi_Vol_Life * 0.5)
    Text = (
        "If imputed as Voluntary Life and was originally classifed as Life assume JUST voluntary and NOT Group Term Life unless pmpm >"
        + str(doubleLim)
    )
    try:
        dt = dt.drop(columns="CT")
    except:
        print("noCT")
    dt["CT"] = dt.groupby("ACK_ID")["Life"].transform("sum")

    # Create a new column CHANGE with 1 if Vol_Life and Life are both 1 and PMPM_EST is less than doubleLim and CT is greater than 1 and Members is less than 1.1 times Part_EOY, else 0
    dt["CHANGE"] = np.where(
        (dt["Vol_Life"] == 1)
        & (dt["Life"] == 1)
        & (dt["PMPM_EST"] < doubleLim)
        & (dt["CT"] > 1)
        & (dt["Members"] < 1.1 * dt["Part_EOY"]),
        1,
        0,
    )

    # Update the Life column with 0 if CHANGE is 1, else keep the previous value
    dt["Life"] = dt.apply(lambda row: 0 if row["CHANGE"] == 1 else row["Life"], axis=1)

    # Update the Rules column with the rule appended if CHANGE is 1, else keep the previous value
    dt["Rules"] = np.where(dt['CHANGE']==0,dt['Rules'],dt['Rules']+rule+ "; ")
    
   
    # Call the executeLobAssign function with some arguments
    dt = executeLobAssign(dt=dt.copy(), lob="Life", NOT=True, rule=rule)
    # Call the ruleCapt function with some arguments
    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description=Text,
        row=886,
    )

    rule = "ExclusiveProductPlan_"
    pmpmCeiling = 70
    Text = (
        "Add specialty flags (Dental, Vision, etc) if it is ONLY a dental/vision/life/disab/inj plan and not a medical contract and PMPMs are < "
        + str(pmpmCeiling)
    )

    dt["PN_WELF_LOBS"] = (
        dt["MEDICAL_PN"]
        + dt["DENTAL_PN"]
        + dt["VISION_PN"]
        + dt["LIFE_PN"]
        + dt["STD_PN"]
        + dt["LTD_PN"]
        + dt["ADD_TRAVEL_PN"]
    )
    if "LEGAL_PN" in dt.columns:
        dt["PN_WELF_LOBS"] = dt["PN_WELF_LOBS"] + dt["LEGAL_PN"]

    products = ["Dental", "Vision", "Life", "STD", "LTD"]
    print("dt col @2831", dt.columns.tolist())
    # INJ
    print("carrier_exact col line 2832:", carrier_exact.columns.tolist())
    for p in products:
        pUp = p.upper()
        carrier_pUp_col = "NOT_" + pUp
        notCarriers = (
            notWelfCarriers
            + carrier_exact[carrier_exact[carrier_pUp_col] == 1][
                "Carrier_String"
            ].tolist()
        )
        dt["CHANGE"] = (
            (dt[p] == 0)
            & (dt["Medical"] == 0)
            & (dt["Other_Vol"] == 0)
            & (dt[pUp + "_PN"] == 1)
            & (dt["PN_WELF_LOBS"] == 1)
            & (~dt["Carrier_String"].isin(notCarriers))
            & (dt["PMPM_EST"] < pmpmCeiling)
        ).astype(int)
        if p in ["LTD", "Life"]:
            dt["CHANGE"] = np.where(
                (dt["Vol_Life"] == 1) | (dt["Sch"] == "C"), 0, dt["CHANGE"]
            )

        thisRule = rule + p
        dt = executeLobAssign(dt=dt.copy(), lob=p, NOT=False, rule=rule)
        rulesDt = ruleCapt(
            DROP=False,
            sub=dt.copy(),
            rulesDt=rulesDt,
            rule=rule,
            Module=fn,
            Yr=y,
            Description=Text,
            row=908,
        )
    #
    # Loop over Medical and Dental

    for p in ["Medical", "Dental"]:
        # Create a rule for LOBAdjust_Not_p_HRA_FSA_HSA_and_LowPMPM
        rule = "LOBAdjust_Not" + p + "_HRA_FSA_HSA_and_LowPMPM"
        Text = "Sch A Only   Not med if low premius amd HRA/FSA/HSA."

        # Assign majors based on p
        if p == "Dental":
            majors = [i.upper() for i in majorDental]
        else:
            majors = [i.upper() for i in majorMedical]

        # Update CHANGE column based on FSA, HRA, HSA, p, FI_RATIO, Stoploss, TPA, Carrier_String, Members and Carrier
        dt["CHANGE"] = np.where(
            ((dt["FSA"] == 1) | (dt["HRA"] == 1) | (dt["HSA"] == 1))
            & (dt[p] == 1)
            & (dt["FI_RATIO"] < 0.35)
            & (dt[["Stoploss", "TPA"]] == 0).all(axis=1)
            & (~dt["Carrier_String"].isin(neverFiStrings))
            & (~((dt["Members"] > 750) & (dt["Carrier"].isin(majors)))),
            1,
            0,
        )

        # Update CHANGE column based on Medical if p is Dental
        if p == "Dental":
            dt["CHANGE"] = np.where(dt["Medical"] == 1, 0, dt["CHANGE"])

        # Execute the lob assign on dt for p
        dt = executeLobAssign(dt=dt, lob=p, NOT=True, ALLOW_NONSTD=True, rule=rule)

        # Capture the rule in rulesDt
        rulesDt = ruleCapt(
            DROP=False,
            sub=dt.copy(),
            rulesDt=rulesDt,
            rule=rule,
            Module=fn,
            Yr=y,
            Description=Text,
            row=923,
        )

    # End for each product
    # Count the LOBS in dt
    dt = countLOBS(dt=dt, fn="Row 850 in" + fn, y=y)

    premLim = 8000
    rule = "LOBAssign_PPOorHMONoLOB_Medical"
    Text = (
        "Switch to Medical if PPO or HMO or (TPA & stand)and missing an LOB & Tot Fees/Premiums >"
        + str(premLim)
    )
    dt.Bundled.value_counts(dropna=False)
    # Create a new column CHANGE with 1 if PPO or HMO or TPA and CarrierNotStd are both 1 and Medical and Medical_Orig are both 0 and Bundled is empty or DummyGroupRecord and Premiums is greater than premLim and Carrier_String is not in notMedStrings, else 0

    dt1 = dt[dt["Bundled"].isin(["", "DummyGroupRecord"])]

    dt["CHANGE"] = np.where(
        (
            (dt["PPO"] == 1)
            | (dt["HMO"] == 1)
            | (dt["TPA"] == 1) & (dt["CarrierNotStd"] == 0)
        )
        & (dt["Medical"] == 0)
        & (dt["Medical_Orig"] == 0)
        & ((dt["Bundled"] == "") | (dt["Bundled"] == "DummyGroupRecord"))
        & (dt["Premiums"] > premLim)
        & (dt["Carrier_String"].isin(notMedStrings) == False),
        1,
        0,
    )

    # Print some summary statistics
    print("\n\nCaught initially by", rule, " ", dt["CHANGE"].sum())
    # Create a new column FL with 1 if CHANGE is 1 and MEDICAL_PN is 0 and Plan_LOBs is not empty, else 0
    dt["FL"] = np.where(
        (dt["CHANGE"] == 1) & (dt["MEDICAL_PN"] == 0) & (dt["Plan_LOBs"] != ""), 1, 0
    )

    print(" reverted bc not MEDICAL_PN ", dt["FL"].sum())

    # Update the CHANGE column with 0 if FL is 1, else keep the previous value
    dt["CHANGE"] = np.where(dt["FL"] == 1, 0, dt.CHANGE)

    # Call the executeLobAssign function with some arguments
    dt = executeLobAssign(dt=dt, lob="Medical", ALLOW_NONSTD=True, rule=rule)

    # Call the ruleCapt function with some arguments
    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description=Text,
        row=943,
    )

    rule = "CNotMedical_LowPMPM"
    Text = "Sch C - Not medical if prems are < 10K and there are other medical contracts with significant prems"
    dt["CT"] = dt.groupby("ACK_ID")["Medical"].transform("sum")
    majorMedical1 = [i.upper() for i in majorMedical]
    dt["CHANGE"] = np.where(
        (dt["Sch"] == "C")
        & (dt["Premiums"] < premLim)
        & (dt["Carrier"].isin(majorMedical1) == False)
        & (dt["HMO"] == 0)
        & ((dt["SI_RATIO"] < 0.25) | (dt["SI_RATIO"] > 2))
        & (dt["CT"] > 1)
        & (dt["Medical"] == 1),
        1,
        0,
    )

    dt = executeLobAssign(dt=dt, lob="Medical", NOT=True, ALLOW_NONSTD=True, rule=rule)
    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description=Text,
        row=952,
    )

    rule = "LOBAssign_Medical_MajorMed_PMPMReasonable; "
    Text = "If no LOBS and a major medical carrier and PMPM reasonable for ASO, assume MEdical"
    dt["MED_CT"] = dt.groupby(["State_HQ", "EIN", "PN"])["Medical"].transform("sum")
    dt["FL"] = np.where(
        (dt["Carrier_String"].isin(alwaysMedStr + [s.upper() for s in majorMedical]))
        & ((dt["TPA"] == 1) | (dt["PPO"] == 1) | (dt["MED_CT"] == 1)),
        1,
        0,
    )
    dt["INSCOPE"] = np.where(
        (dt["Medical"] == 0)
        & (dt["Bundled"].isin(["", "DummyGroupRecord"]))
        & (~dt["Carrier_String"].isin(notMedStrings))
        & (dt["MEDICAL_PN"] == 1)
        & (dt["CarrierNotStd"] == 0),
        1,
        0,
    )

    print(dt["INSCOPE"].mean())

    # Update INSCOPE column based on Sch, FL, PMPM_EST and pmpm_aso_Medical
    dt["INSCOPE"] = np.where(
        (dt["Sch"] == "C")
        & (dt["FL"] == 0)
        & (
            (dt["PMPM_EST"] < 0.15 * pmpm_aso_Medical)
            | (dt["PMPM_EST"] > 10 * pmpm_aso_Medical)
        ),
        0,
        dt["INSCOPE"],
    )

    dt["INSCOPE"] = np.where(
        (dt["Sch"] == "A")
        & (dt["FL"] == 0)
        & (
            (dt["PMPM_EST"] < 0.4 * pmpm_fi_Medical)
            | (dt["PMPM_EST"] > 4 * pmpm_fi_Medical)
        ),
        0,
        dt["INSCOPE"],
    )

    # Update INSCOPE column based on Sch, FL, PMPM_EST and pmpm_fi_Medical
    dt["CHANGE"] = np.where(
        (dt.INSCOPE == 1)
        & ((dt.TPA == 1) | (dt.MED_CT == 0) | (dt.PPO == 1) | (dt.Premiums > 5000)),
        1,
        dt.CHANGE,
    )
    # Update CHANGE column based on INSCOPE, TPA, MED_CT, PPO and Premiums
    # Capture the rule in rulesDt
    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description=Text,
        row=967,
    )

    print(
        "Time taken to get to line 3030:",
        time.time() - start_time_impute_lob,
        "seconds",
    )
    # Execute the lob assign on dt for Medical
    dt = executeLobAssign(dt=dt, lob="Medical", ALLOW_NONSTD=True, rule=rule)

    if y == 2017:
        # Assign the dt dataframe filtered by Bundled, Medical and Benefit_Oth columns to a global variable noLOBBenText
        noLOBBenText = dt[
            (dt["Bundled"] == "") & (dt["Medical"] == 0) & (dt["Benefit_Oth"] != "")
        ]
        print(
            "\nObs saved to global envt as noLOBBenText (no LOB but has ben text) ",
            len(noLOBBenText),
        )

    rule = "SchC_Drop_NoLOB_NotStdOrPremsSmall"
    Text = "Drop ScC if carrier nos tanardized and no LOBs could be assigned "
    dt["MED_CT"] = dt.groupby(["State_HQ", "EIN", "PN", "Year"])["Medical"].transform(
        "sum"
    )
    # Create a new column DROP with 1 if Sch is C and either CarrierNotStd is 1 or Premiums is less than 1 and Bundled is empty and Medical is 0 and Carrier is not in uppercase majorMedical and MED_CT is not 0, else 0
    majorMedical1 = [s.upper() for s in majorMedical]
    len(majorMedical1)
    dt["DROP"] = np.where(
        (dt["Sch"] == "C")
        & ((dt["CarrierNotStd"] == 1) | (dt["Premiums"] < 1))
        & (dt["Bundled"] == "")
        & (dt["Medical"] == 0)
        & ~((dt["Carrier"].isin(majorMedical1) == True) & (dt["MED_CT"] == 0)),
        1,
        0,
    )

    # Call the ruleCapt function with some arguments
    rulesDt = ruleCapt(
        DROP=True,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=971,
    )
    # Call the executeDropRuleDT function with some arguments
    dt = executeDropRuleDT(dt=dt, rule=rule)

    rule = "LOBAssign_LifeOrDisab; "
    # Create a new column LIFE_CT with the sum of Life within each group of State_HQ, EIN, PN and Year
    dt["LIFE_CT"] = dt.groupby(["State_HQ", "EIN", "PN", "Year"])["Life"].transform(
        "sum"
    )
    # Create a new column LTD_CT with the sum of Life within each group of State_HQ, EIN, PN and Year
    dt["LTD_CT"] = dt.groupby(["State_HQ", "EIN", "PN", "Year"])["Life"].transform(
        "sum"
    )
    # Create a new column STD_CT with the sum of Life within each group of State_HQ, EIN, PN and Year
    dt["STD_CT"] = dt.groupby(["State_HQ", "EIN", "PN", "Year"])["Life"].transform(
        "sum"
    )
    # Create a new column FL with 1 if Bundled is empty or DummyGroupRecord and Medical is 0, else 0
    dt["FL"] = np.where(
        (dt["Bundled"].isin(["", "DummyGroupRecord"])) & (dt["Medical"] == 0), 1, 0
    )

    # Call the countLOBS function with some arguments
    dt = countLOBS(dt=dt, fn="@row 840 in" + fn, y=y)

    # Filter the dt dataframe by Bundled and Medical columns
    te = dt[(dt["Bundled"] == "") & (dt["Medical"] == 0)]
    print("\nObs still missing bundle/LOB at end fo imputeLOB ", len(te))
    print(te["Sch"].value_counts())

    rule = "MissingPart_EOY_ifTOTAL_PARTMissing_UseMaxLifeMembers; "

    byVars = ["State_HQ", "EIN", "Year"]
    byVarsPn = byVars + ["PN"]

    dt["KEY"] = np.where(
        (dt["Life"] == 1) | (dt["LTD"] == 1) | (dt["STD"] == 1), dt["Members"], 0
    )
    dt["MEMBMAX"] = dt.groupby(byVarsPn)["KEY"].transform("max")
    dt["MEMBMAX2"] = dt.groupby(byVarsPn)["Members"].transform("max")
    dt["MEMBMAX"] = np.where(
        (dt.MEMBMAX < 100) & (dt.MEMBMAX2 >= 100), dt.MEMBMAX2, dt.MEMBMAX
    )
    dt["CHANGE"] = np.where(
        (dt["Part_EOY"] == 0) & (dt["PART_MAX"] == 0) & (dt["MEMBMAX"] > 0), 1, 0
    )
    dt["Part_EOY"] = np.where(dt["CHANGE"] == 1, dt["MEMBMAX"], dt["Part_EOY"])
    dt["PART_MAX"] = np.where(dt["CHANGE"] == 1, dt["Part_EOY"], dt["PART_MAX"])
    dt["Rules"] = np.where(dt["CHANGE"] == 0, dt["Rules"], dt["Rules"] + rule)

    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=986,
    )
    dt.rename(columns={"PMPM_EST": "PMPM"}, inplace=True)
    dt["MID_COMPOSITE"] = np.where(
        (dt["Funding"].isin(["LF", "FI"])) | (dt["Stoploss"] == 1),
        dt["PMPM_MED_FI"],
        dt["PMPM_MED_SI"],
    )

    rule = "CMissingPrems_useDefaultPMPM"

    Text = "Sched C: if missing premiums derive from default SI PMPM assumption"
    dt["CHANGE"] = np.where(
        (dt["Sch"] == "C") & (dt["Premiums"] < 1) & (dt["Members"] > 0), 1, 0
    )

    print("dt col line 3162:", dt.columns.tolist())
    print("\nCaught by ", rule, " ", dt["CHANGE"].sum())
    dt["PMPM"] = dt["PMPM"].astype('float64')
    executePMPMChangeDefault(MEMB=False, PREM=True, dt=dt, y=y, rule=rule)
    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=1015,
    )

    print("\n\nSTART MISSING MEMBER IMPUTATION FOR SCHEDULE A ACCOUNTS ")

    te = dt[(dt["Members"] == 0) & (dt["Sch"] == "A")]
    print("\nSch A obs with missing members @1030 ", te.shape[0])

    rule = "MissingPartAndMembers_Impute_PMPM_Members_Part_FromPrems"
    Text = "Sched A: if participants and members are missing, use default PMPM and premiums to back out members"

    dt["CHANGE"] = np.where(
        (dt["PART_MAX"] < 10)
        & (dt["Members"] < 10)
        & (~dt["Bundled"].isin(["", "DummyGroupRecord"]))
        & (dt["Premiums"] > 5000)
        & (dt["Sch"] == "A"),
        1,
        0,
    )
    print("\nCaught by", rule, " ", dt["CHANGE"].sum())

    print("dt col line 3193:", dt.columns.tolist())
    # NOTE: added try/except here

    dt = executePMPMChangeDefault(MEMB=True, PREM=False, dt=dt, y=y, rule=rule)
    dt["Part_EOY"] = np.where(
        (dt["CHANGE"] == 1) & (dt["Members"] > 0), dt["Members"], dt["Part_EOY"]
    )
    dt["PART_MAX"] = np.where(dt["CHANGE"] == 1, dt["Part_EOY"], dt["PART_MAX"])

    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description="",
        row=1031,
    )

    te = dt[(dt["Members"] == 0) & (dt["Sch"] == "A")]
    print("\nSch A obs with missing members @1042 ", te.shape[0])

    rule = "MissingMembAndPrems_SetToExpectedPMPMUsePART_MAX"
    Text = "Sch A: If missinga bunch of stuff but only contract for STATExein also drop if this creates dups"

    dt["CHANGE"] = np.where(
        (dt["Sch"] == "A")
        & (dt["MEMBERS_OR_SUBSCRIBERS"] < 3)
        & (dt["Premiums"] < 3)
        & (dt["PART_MAX"] > 20),
        1,
        0,
    )

    dt["MEMB_TO_PART_EST"] = np.where(
        dt["MEMB_TO_PART_EST"] < 0.2, 0.2, dt["MEMB_TO_PART_EST"]
    )
    # NOTE: throwing errors...
    dt["PMPM"] = np.where(dt["CHANGE"] == 1, dt["PMPM_MED_FI"], dt["PMPM"])
    dt["Members"] = np.where(
        dt["CHANGE"] == 1, dt["PART_MAX"] * dt["MEMB_TO_PART_EST"], dt["Members"]
    )

    dt["Premiums"] = np.where(
        dt["CHANGE"] == 1, lasj_round(dt["Members"] * 12 * dt["PMPM"]), dt["Premiums"]
    )

    dt["Rules"] = np.where(
        dt["CHANGE"] == 0,
        dt["Rules"],
        dt["Rules"]
        + rule
        + ".InitPREMS"
        + dt["Premiums_Orig"].astype(str)
        + ".InitMemb"
        + dt["MEMBERS_OR_SUBSCRIBERS"].astype(str)
        + "; ",
    )

    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description=Text,
        row=1045,
    )
    rule = "MissingMemb_ImputeWithParticipants"
    Text = "Sch A: if missing members bbut not missing participants, leverage the MEMB_TO_PART_EST field"

    dt["DEN"] = lasj_round(
        (dt["Duration"] / 30.4) * (dt["PART_MAX"] * dt["MEMB_TO_PART_EST"]), 1
    )

    dt["FL"] = np.where(
        (dt["Sch"] == "A")
        & (dt["Members"] == 0)
        & (dt["PART_MAX"] > 1)
        & (dt["MEMB_TO_PART_EST"] > 0.01),
        1,
        0,
    )

    dt["CHANGE"] = np.where((dt["FL"] == 1) & (dt["DEN"] > 1), 1, 0)

    dt["FL"] = np.where(
        (dt["FL"] == 1) & (dt["CHANGE"] == 0) & (dt["Duration"] < 10), 1, 0
    )

    dt["DEN"] = np.where(
        (dt["FL"] == 1) & (dt["DEN"] < 1),
        lasj_round(12 * dt["PART_MAX"] * dt["MEMB_TO_PART_EST"], 1),
        dt["DEN"],
    )

    print(
        "\nTotal caught by ",
        rule,
        "@1053 w/Duration > 30 ",
        dt["CHANGE"].sum(),
        "; those with Duration <10 ",
        dt["FL"].sum(),
    )

    dt["CHANGE"] = np.where(dt["FL"] == 1, 1, dt["CHANGE"])

    dt["Members"] = np.where(
        dt["CHANGE"] == 1,
        lasj_round(dt["MEMB_TO_PART_EST"] * dt["PART_MAX"]),
        dt["Members"],
    )

    dt["PMPM"] = np.where(
        (dt["CHANGE"] == 1) & (dt["DEN"] > 0),
        lasj_round(dt["Premiums"] / dt["DEN"]),
        dt["PMPM"],
    )

    dt["Rules"] = np.where(dt["CHANGE"] == 0, dt["Rules"], dt["Rules"] + rule + "; ")

    rulesDt = ruleCapt(
        DROP=False,
        sub=dt.copy(),
        rulesDt=rulesDt,
        rule=rule,
        Module=fn,
        Yr=y,
        Description=Text,
        row=1061,
    )

    # QA/TESTING
    # NOTE: this checkForBads function looks wonky in R. References dt and dtCheck

    checkForBads(dtCheck=dt.copy(), txt="@1075 imputeLob", y=y)

    te = dt[(dt["Members"] == 0) & (dt["Sch"] == "A")]
    print("\nSchA obs with missing members @950 ", te.shape[0])

    # printFITotals(dt=dt, y=y, EXEMPT=False)
    dt = cleanupJunk(dt=dt, otherJunk=["Medical_Orig", "STD_CT", "LTD_CT", "MED_CT", "LIFE_CT"])

    dt["Rules"] = np.where((dt["Rules"] != "") & (dt["Rules"].str[0] == " "), dt["Rules"].str[1:100],dt["Rules"])
    # save dt as a pickle file in the pitByYearDir/y/ directory 
    for i in dt.columns:
        if dt[i].dtype=='object':
            dt[i]=dt[i].astype('category')
   
    dt.to_pickle(os.path.join(pitByYearDir, str(y), outFile + ".pkl"))
    
    
    
    # save rulesDt as a pickle file in the rules_dir directory
    rulesDt.to_pickle(rules_dir + outFile + "_rules_"+ str(y)+".pkl")
    print("Completed. Time taken:", time.time() - start_time_impute_lob, "seconds")
    # print a message with the end time, file name and number of rows in dt
    txt = f"\nEnd {fn}. Obs saved in {outFile} {dt.shape[0]}{dot20end}"
    print(txt)

    #  closeAllConnections()

    # print the message again if SINK is True
    if SINK:
        print(txt)


# end impute_lob function