Kaggle competition: Avito Demand Prediction Challenge
<SOS> metric: RMSE Kaggle data source: avito-demand-prediction-challenge<create_dataframe>
print(os.listdir(".. /input"))
df = pd.read_csv(".. /input/avito-demand-prediction/train.csv")
 test = pd.read_csv(".. /input/avito-demand-prediction/test.csv")
 df_sorted = df.sort_values(by = ["image"])
 test_sorted = test.sort_values(by = ["image"] )
def counsruct(dataframe,test = False):
PACIENT = dataframe["Patient"].unique()
TRAIN_NEW = pd.DataFrame()
for ID in tqdm(PACIENT):
data = dataframe[dataframe.Patient == ID].copy()
data.reset_index(inplace=True,drop=True)
 week_start, FVC_start, Percent_kt,d = data.loc[0,["Weeks","FVC","Percent","dir"]].values
DIR=d
DIR_DCM = DIR+ID+"/"
d = os.listdir(DIR_DCM)
 d = sorted(os.listdir(DIR_DCM), key=lambda v:int(v.split('.')[0]))
count_slice = len(d)
 d = {i+1:dcm for i,dcm in enumerate(d)}
center = len(d)//2
c=center-(center*40//100)
 arr_slice = [c]
count_repeat = len(arr_slice)
 d = np.array([[d[i],int(i)] for j in range(data.shape[0])for i in arr_slice])
 d = pd.DataFrame(d,columns=["dcm","num_slice"])
 d["num_slice"] = d["num_slice"].astype("int")
 data = data.loc[data.index.repeat(count_repeat)].reset_index(drop=True)
 data[["dcm","num_slice"]] = d[["dcm","num_slice"]]
data["count_slice"] = count_slice
data["week_kt"] = week_start
data["FVC_kt"] = FVC_start
data["Percent_kt"] = Percent_kt
TRAIN_NEW = pd.concat([TRAIN_NEW, data], ignore_index=True)
 return TRAIN_NEW
TRAIN_C = counsruct(TRAIN)
 TEST_C = counsruct(TEST,True)
 TEST_C["Count_weks"] = TEST_C["week_predict"]-TEST_C["week_kt"]
TRAIN_C["Count_weks"] = TRAIN_C["Weeks"]-TRAIN_C["week_kt"] 
<feature_engineering>
tr_1 = pd.read_csv(".. /input/train-data-image-blurrness/1.csv")
 tr_2 = pd.read_csv(".. /input/train-data-image-blurrness/2.csv")
 tr_3 = pd.read_csv(".. /input/train-data-image-blurrness/3.csv")
 tr_4 = pd.read_csv(".. /input/train-data-image-blurrness/4.csv")
 tr_5 = pd.read_csv(".. /input/train-data-image-blurrness/5.csv")
 tr_6 = pd.read_csv(".. /input/train-data-image-blurrness/6.csv")
 tr_7 = pd.read_csv(".. /input/train-data-image-blurrness/7.csv")
 tr_8 = pd.read_csv(".. /input/train-data-image-blurrness/8.csv")
 tr_9 = pd.read_csv(".. /input/train-data-image-blurrness/9.csv")
 tr_10 = pd.read_csv(".. /input/train-data-image-blurrness/10.csv")
 tr_11 = pd.read_csv(".. /input/train-data-image-blurrness/11(12_13.5 ).csv")
 tr_12 = pd.read_csv(".. /input/train-data-image-blurrness/last.csv")
 frames = [tr_1, tr_2, tr_3, tr_4, tr_5, tr_6, tr_7, tr_8, tr_9, tr_10, tr_11, tr_12]
new = pd.concat(frames)
 new["File"] = new["File"].apply(lambda x : x.split("/")[-1].split(".")[0])
 new = new.sort_values(by = ["File"])
 scores = list(new["Score"].values)+ [-1] *(len(df)-len(new))
df_sorted["image_blurrness_score"] = scores
df = df_sorted.sort_index()
<create_dataframe>
te_1 = pd.read_csv(".. /input/image-blurrness-test/test_1.csv")
 te_2 = pd.read_csv(".. /input/image-blurrness-test/test_2.csv")
 te_3 = pd.read_csv(".. /input/image-blurrness-test/test_3.csv")
 te_4 = pd.read_csv(".. /input/image-blurrness-test/test_4.csv")
 te_5 = pd.read_csv(".. /input/image-blurrness-test/test_5.csv")
 frames_te = [te_1, te_2, te_3, te_4, te_5]
new_te = pd.concat(frames_te)
 new_te["File"] = new_te["File"].apply(lambda x : x.split("/")[-1].split(".")[0])
 new_te = new_te.sort_values(by = ["File"])
 scores_te = list(new_te["Score"].values)+ [-1] *(len(test)-len(new_te))
test_sorted["image_blurrness_score"] = scores_te
test = test_sorted.sort_index()
r = 1
e=70
def custom_data(dataframe):
dataframe["FVC_n"] =(dataframe["FVC_kt"]*100/dataframe["Percent_kt"])
 for i in range(r,e):
name ="FVC_mean"+str(i)
 name2 = "FVC_custom"+str(i)
 dataframe[name] =(dataframe["FVC_n"]-dataframe["FVC_kt"])/(30*i)
 dataframe[name2] =(dataframe["FVC_kt"]-(dataframe["Count_weks"])*dataframe[name])-(( dataframe["Count_weks"]+90))
return dataframe
TRAIN_C=custom_data(TRAIN_C)
 TEST_C=custom_data(TEST_C)
 name = ["FVC_custom"+str(i)for i in range(r,e)]
TEST_C["FVC_PRE"] = TEST_C[name[10:]].mean(axis=1)
 TRAIN_C["FVC_PRE"] = TRAIN_C[name[10:]].mean(axis=1)
 TEST_C["FVC_PRE2"] = TEST_C["FVC_PRE"]**2
TRAIN_C["FVC_PRE2"] = TRAIN_C["FVC_PRE"]**2
TEST_C["FVC_n2"] = TEST_C["FVC_n"]**2
TRAIN_C["FVC_n2"] = TRAIN_C["FVC_n"]**2
TEST_C["r1"] = TEST_C["FVC_n"] - TEST_C["FVC_PRE"]
TEST_C["r1mean"] = TEST_C[["FVC_n","FVC_PRE"]].mean(axis=1)
 TEST_C["r2"] = TEST_C[["FVC_n","FVC_PRE"]].std(axis=1)
 TRAIN_C["r1"] = TRAIN_C["FVC_n"] - TRAIN_C["FVC_PRE"]
TRAIN_C["r1mean"] = TRAIN_C[["FVC_n","FVC_PRE"]].mean(axis=1)
 TRAIN_C["r2"] = TRAIN_C[["FVC_n","FVC_PRE"]].std(axis=1)
 TRAIN_C[["Female","Male","Currently smokes","Ex-smoker","Never smoked"]]=0
TEST_C[["Female","Male","Currently smokes","Ex-smoker","Never smoked"]]=0
def calculate_height(row):
if row['Sex'] == 'Male':
return row['FVC_kt'] /(27.63 - 0.112 * row['Age'])
 else:
return row['FVC_kt'] /(21.78 - 0.101 * row['Age'])
 def calculate_all(row):
if row['Sex'] == 'Male':
row['Male'] = 1
else:
row['Female']=1
if row['SmokingStatus'] == "Currently smokes":
row['Currently smokes']= 1
if row['SmokingStatus'] == "Ex-smoker":
row['Ex-smoker']= 1
if row['SmokingStatus'] == "Never smoked":
row['Never smoked']= 1
return row
TRAIN_C['Height'] = TRAIN_C.apply(calculate_height, axis=1)
 TEST_C['Height'] = TEST_C.apply(calculate_height, axis=1)
 TRAIN_C = TRAIN_C.apply(calculate_all, axis=1)
 TEST_C = TEST_C.apply(calculate_all, axis=1)
 TRAIN_C = TRAIN_C.drop(columns=['Sex', 'SmokingStatus'])
 TEST_C = TEST_C.drop(columns=['Sex', 'SmokingStatus'])
<categorify>
df_train= df
df_test = test
def construct_binned(dataframe_train,dataframe_test,column,count,combi=True):
col = dataframe_train[column].unique().tolist()
col.extend(dataframe_test[column].unique().tolist())
 all_col = np.array(col)
 bins = np.linspace(all_col.min() ,all_col.max() ,count)
 col_data_train = dataframe_train[column].values.reshape(-1,1)
 col_data_test = dataframe_test[column].values.reshape(-1,1)
 witch_bin = np.digitize(col_data_train,bins)
 witch_bin2 = np.digitize(col_data_test,bins)
 encoder = OneHotEncoder(sparse=False)
 encoder.fit(witch_bin)
 col_binned = encoder.transform(witch_bin)
 col_binned2 = encoder.transform(witch_bin2)
 col_binned_combi = np.hstack([col_data_train,col_binned])
 col_binned2_combi = np.hstack([col_data_test,col_binned2])
 set1 = col_binned_combi
set2 = col_binned2_combi
if not combi:
set1 = col_binned
set2 = col_binned2
name_col_binned = [column+"_binned"+str(i)for i in range(set1.shape[1])]
dataframe_train[name_col_binned] = set1
dataframe_test[name_col_binned] = set2
return dataframe_train, dataframe_test,name_col_binned
TRAIN_C,TEST_C,binned_age = construct_binned(TRAIN_C,TEST_C,"Age",60,True)
 TRAIN_C,TEST_C,binned_height = construct_binned(TRAIN_C,TEST_C,"Height",5,True)
 TRAIN_C,TEST_C,binned_Percent_kt = construct_binned(TRAIN_C,TEST_C,"Percent_kt",60,True)
 TRAIN_C,TEST_C,binned_FVC_kt = construct_binned(TRAIN_C,TEST_C,"FVC_kt",60,True)
 TRAIN_C,TEST_C,binned_Count_week_kt = construct_binned(TRAIN_C,TEST_C,"week_kt",15,True)
 TRAIN_C,TEST_C,binned_FVC_n = construct_binned(TRAIN_C,TEST_C,"FVC_n",60,True)
 TRAIN_C,TEST_C,binned_count_slice = construct_binned(TRAIN_C,TEST_C,"count_slice",15,True )<categorify>
parent_category_name_map = {"Личные вещи" : "Personal belongings",
"Для дома и дачи" : "For the home and garden",
"Бытовая электроника" : "Consumer electronics",
"Недвижимость" : "Real estate",
"Хобби и отдых" : "Hobbies & leisure",
"Транспорт" : "Transport",
"Услуги" : "Services",
"Животные" : "Animals",
"Для бизнеса" : "For business"}
Height = TRAIN_C["Height"].unique().tolist()
Height.extend(TEST_C["Height"].unique().tolist())
 all_Height = np.unique(Height)
 bins = np.linspace(all_Height.min() ,all_Height.max() ,5)
 witch_bin = np.digitize(TRAIN_C.Height,bins)
 witch_bin2 = np.digitize(TEST_C.Height,bins)
 encoder = OneHotEncoder(sparse=False)
 encoder.fit(witch_bin.reshape(-1, 1))
Height_binned = encoder.transform(witch_bin.reshape(-1, 1))
Height_binned2 = encoder.transform(witch_bin2.reshape(-1, 1))
name_height_binned = ["Height_binned"+str(i)for i in range(Height_binned.shape[1])]
TRAIN_C[name_height_binned] = Height_binned
TEST_C[name_height_binned] = Height_binned2<categorify>
region_map = {"Свердловская область" : "Sverdlovsk oblast",
"Самарская область" : "Samara oblast",
"Ростовская область" : "Rostov oblast",
"Татарстан" : "Tatarstan",
"Волгоградская область" : "Volgograd oblast",
"Нижегородская область" : "Nizhny Novgorod oblast",
"Пермский край" : "Perm Krai",
"Оренбургская область" : "Orenburg oblast",
"Ханты-Мансийский АО" : "Khanty-Mansi Autonomous Okrug",
"Тюменская область" : "Tyumen oblast",
"Башкортостан" : "Bashkortostan",
"Краснодарский край" : "Krasnodar Krai",
"Новосибирская область" : "Novosibirsk oblast",
"Омская область" : "Omsk oblast",
"Белгородская область" : "Belgorod oblast",
"Челябинская область" : "Chelyabinsk oblast",
"Воронежская область" : "Voronezh oblast",
"Кемеровская область" : "Kemerovo oblast",
"Саратовская область" : "Saratov oblast",
"Владимирская область" : "Vladimir oblast",
"Калининградская область" : "Kaliningrad oblast",
"Красноярский край" : "Krasnoyarsk Krai",
"Ярославская область" : "Yaroslavl oblast",
"Удмуртия" : "Udmurtia",
"Алтайский край" : "Altai Krai",
"Иркутская область" : "Irkutsk oblast",
"Ставропольский край" : "Stavropol Krai",
"Тульская область" : "Tula oblast"}
FVC_kt_train = TRAIN_C["FVC_kt"].unique().tolist()
FVC_kt_train.extend(TEST_C["FVC_kt"].unique().tolist())
 all_FVC_kt = np.unique(FVC_kt_train)
 bins = np.linspace(all_FVC_kt.min() ,all_FVC_kt.max() ,11)
 witch_bin = np.digitize(TRAIN_C.FVC_kt,bins)
 witch_bin2 = np.digitize(TEST_C.FVC_kt,bins)
 encoder = OneHotEncoder(sparse=False)
 encoder.fit(witch_bin.reshape(-1, 1))
FVC_binned = encoder.transform(witch_bin.reshape(-1, 1))
FVC_binned2 = encoder.transform(witch_bin2.reshape(-1, 1))
name_bin_fvckt = ["FVC_KT_bin"+str(i)for i in range(FVC_binned.shape[1])]
TRAIN_C[name_bin_fvckt] = FVC_binned
TEST_C[name_bin_fvckt] = FVC_binned2<categorify>
category_map = {"Одежда, обувь, аксессуары":"Clothing, shoes, accessories",
"Детская одежда и обувь":"Children's clothing and shoes",
"Товары для детей и игрушки":"Children's products and toys",
"Квартиры":"Apartments",
"Телефоны":"Phones",
"Мебель и интерьер":"Furniture and interior",
"Предложение услуг":"Offer services",
"Автомобили":"Cars",
"Ремонт и строительство":"Repair and construction",
"Бытовая техника":"Appliances",
"Товары для компьютера":"Products for computer",
"Дома, дачи, коттеджи":"Houses, villas, cottages",
"Красота и здоровье":"Health and beauty",
"Аудио и видео":"Audio and video",
"Спорт и отдых":"Sports and recreation",
"Коллекционирование":"Collecting",
"Оборудование для бизнеса":"Equipment for business",
"Земельные участки":"Land",
"Часы и украшения":"Watches and jewelry",
"Книги и журналы":"Books and magazines",
"Собаки":"Dogs",
"Игры, приставки и программы":"Games, consoles and software",
"Другие животные":"Other animals",
"Велосипеды":"Bikes",
"Ноутбуки":"Laptops",
"Кошки":"Cats",
"Грузовики и спецтехника":"Trucks and buses",
"Посуда и товары для кухни":"Tableware and goods for kitchen",
"Растения":"Plants",
"Планшеты и электронные книги":"Tablets and e-books",
"Товары для животных":"Pet products",
"Комнаты":"Room",
"Фототехника":"Photo",
"Коммерческая недвижимость":"Commercial property",
"Гаражи и машиноместа":"Garages and Parking spaces",
"Музыкальные инструменты":"Musical instruments",
"Оргтехника и расходники":"Office equipment and consumables",
"Птицы":"Birds",
"Продукты питания":"Food",
"Мотоциклы и мототехника":"Motorcycles and bikes",
"Настольные компьютеры":"Desktop computers",
"Аквариум":"Aquarium",
"Охота и рыбалка":"Hunting and fishing",
"Билеты и путешествия":"Tickets and travel",
"Водный транспорт":"Water transport",
"Готовый бизнес":"Ready business",
"Недвижимость за рубежом":"Property abroad"}
FVC_PRE_train = TRAIN_C["FVC_PRE"].unique().tolist()
FVC_PRE_train.extend(TEST_C["FVC_PRE"].unique().tolist())
 all_FVC_PRE_kt = np.unique(FVC_PRE_train)
 bins2 = np.linspace(all_FVC_PRE_kt.min() ,all_FVC_PRE_kt.max() ,5)
 witch_bin2 = np.digitize(TRAIN_C.FVC_PRE,bins2)
 witch_bin22 = np.digitize(TEST_C.FVC_PRE,bins2)
 encoder = OneHotEncoder(sparse=False)
 encoder.fit(witch_bin.reshape(-1, 1))
FVC_PRE_binned = encoder.transform(witch_bin2.reshape(-1, 1))
FVC_PRE_binned2 = encoder.transform(witch_bin22.reshape(-1, 1))
name_bin_pre = ["FVC_PRE_bin"+str(i)for i in range(FVC_binned.shape[1])]
TRAIN_C[name_bin_pre] = FVC_PRE_binned
TEST_C[name_bin_pre] = FVC_PRE_binned2 
<feature_engineering>
params_top35_map = {'Женская одежда':"Women's clothing",
'Для девочек':'For girls',
'Для мальчиков':'For boys',
'Продам':'Selling',
'С пробегом':'With mileage',
'Аксессуары':'Accessories',
'Мужская одежда':"Men's Clothing",
'Другое':'Other','Игрушки':'Toys',
'Детские коляски':'Baby carriages',
'Сдам':'Rent',
'Ремонт, строительство':'Repair, construction',
'Стройматериалы':'Building materials',
'iPhone':'iPhone',
'Кровати, диваны и кресла':'Beds, sofas and armchairs',
'Инструменты':'Instruments',
'Для кухни':'For kitchen',
'Комплектующие':'Accessories',
'Детская мебель':"Children's furniture",
'Шкафы и комоды':'Cabinets and chests of drawers',
'Приборы и аксессуары':'Devices and accessories',
'Для дома':'For home',
'Транспорт, перевозки':'Transport, transportation',
'Товары для кормления':'Feeding products',
'Samsung':'Samsung',
'Сниму':'Hire',
'Книги':'Books',
'Телевизоры и проекторы':'Televisions and projectors',
'Велосипеды и самокаты':'Bicycles and scooters',
'Предметы интерьера, искусство':'Interior items, art',
'Другая':'Other','Косметика':'Cosmetics',
'Постельные принадлежности':'Bed dress',
'С/х животные' :'Farm animals','Столы и стулья':'Tables and chairs'}
def calculate_FVC(row):
if row['Weeks'] == row['week_kt']:
row['FVC_PRE'] = row['FVC']
return row
TRAIN_C = TRAIN_C.apply(calculate_FVC, axis=1 )<categorify>
df_train['region_en'] = df_train['region'].apply(lambda x : region_map[x])
 df_train['parent_category_name_en'] = df_train['parent_category_name'].apply(lambda x : parent_category_name_map[x])
 df_train['category_name_en'] = df_train['category_name'].apply(lambda x : category_map[x])
 del df_train['region']
del df_train['parent_category_name']
del df_train['category_name']
df_train
df_test['region_en'] = df_test['region'].apply(lambda x : region_map[x])
 df_test['parent_category_name_en'] = df_test['parent_category_name'].apply(lambda x : parent_category_name_map[x])
 df_test['category_name_en'] = df_test['category_name'].apply(lambda x : category_map[x])
 del df_test['region']
del df_test['parent_category_name']
del df_test['category_name']
df_test
df_train
Patient = LabelEncoder()
train_pac = TRAIN_C["Patient"].unique().tolist()
train_pac.extend(TEST_C["Patient"].unique().tolist())
 all_pacient = np.unique(train_pac)
 Patient.fit(all_pacient)
 def LE(dataframe, val=False,dense=False):
dataframe["patiet_id"] = Patient.transform(dataframe["Patient"])
 col = ["Patient","dcm"]
if not val:
col.extend(["FVC","Weeks"])
 if val:
col.extend(["week_predict"])
 col.extend(name_bin_pre)
 col.extend(name_height_binned)
 col.extend(binned_age)
 col.extend(binned_FVC_kt)
 col.extend(binned_Percent_kt)
 col.extend(binned_FVC_n)
 col.extend(["FVC_PRE","FVC_PRE2","count_slice","week_kt","Count_weks",'Height',"Currently smokes","Ex-smoker","Never smoked","Female","Male","r1","r2","r1mean"])
 dataframe = dataframe[col]
dataframe['dcm'] = dataframe.agg('{0[Patient]}/{0[dcm]}'.format, axis=1)
 return dataframe
TRAIN2 = LE(TRAIN_C.copy())
 TEST2 = LE(TEST_C.copy() ,True)
<compute_test_metric>
regional = pd.read_csv(".. /input/regionaldata/regional.csv", index_col = [0])
 rDense = regional["Density_of_region(km2)"]
rRural = regional["Rural_%"]
rTime_zone = regional["Time_zone"]
rPopulation = regional["Total_population"]
rUrban = regional["Urban%"]
reg_index = np.array([regional.index[i].lower() for i in range(len(regional)) ])
 rDense.index = reg_index
rRural.index = reg_index
rTime_zone.index = reg_index
rPopulation.index = reg_index
rUrban.index = reg_index
df_region = df_train["region_en"]
reg_dense = np.array([rDense[df_region[i].lower() ] for i in range(len(df_train)) ])
 reg_rural = np.array([rRural[df_region[i].lower() ] for i in range(len(df_train)) ])
 reg_Time_zone = np.array([rTime_zone[df_region[i].lower() ] for i in range(len(df_train)) ])
 reg_Population = np.array([rPopulation[df_region[i].lower() ] for i in range(len(df_train)) ])
 reg_Urban = np.array([rUrban[df_region[i].lower() ] for i in range(len(df_train)) ])
 df_train["reg_dense"] = reg_dense
df_train["rural"] = reg_rural
df_train["reg_Time_zone"] = reg_Time_zone
df_train["reg_Population"] = reg_Population
df_train["reg_Urban"] = reg_Urban
df_train
reg_dense = np.array([rDense[df_region[i].lower() ] for i in range(len(df_test)) ])
 reg_rural = np.array([rRural[df_region[i].lower() ] for i in range(len(df_test)) ])
 reg_Time_zone = np.array([rTime_zone[df_region[i].lower() ] for i in range(len(df_test)) ])
 reg_Population = np.array([rPopulation[df_region[i].lower() ] for i in range(len(df_test)) ])
 reg_Urban = np.array([rUrban[df_region[i].lower() ] for i in range(len(df_test)) ])
 df_test["reg_dense"] = reg_dense
df_test["rural"] = reg_rural
df_test["reg_Time_zone"] = reg_Time_zone
df_test["reg_Population"] = reg_Population
df_test["reg_Urban"] = reg_Urban
df_train
def laplace_log_likelihood(actual_fvc, predicted_fvc, confidence, return_values = False):

 sd_clipped = np.maximum(confidence, 70)
 delta = np.minimum(np.abs(actual_fvc - predicted_fvc), 1000)
 metric = - np.sqrt(2)* delta / sd_clipped - np.log(np.sqrt(2)* sd_clipped)
 if return_values:
return metric
else:
return np.mean(metric )<prepare_x_and_y>
new_df_train = df_train.copy()
new_df_test = df_test.copy()
del new_df_test["image"]
del new_df_train["image"]
del new_df_test["activation_date"]
del new_df_train["activation_date"]
train_split = train.copy()
test_split = validation.copy()
X_train = train_split[train_split.columns.tolist() [3:]].copy()
X_val= test_split[test_split.columns.tolist() [3:]].copy()
X_val2= test_split[test_split.columns.tolist() [3:]].copy()
X_end = TRAIN2[train_split.columns.tolist() [3:]].copy()
X_end2 = TRAIN2[train_split.columns.tolist() [3:]].copy()
Y_end = TRAIN2["FVC"].copy()
Y_train = train_split["FVC"].copy()
Y_train2 = Y_train- train_split["FVC_PRE"]
Y_train=Y_train2.abs()
Y_val = test_split["FVC"].copy()
Y_val2 = Y_val- test_split["FVC_PRE"]
Y_val2 = Y_val2.abs()
X_test = TEST2[TEST2.columns.tolist() [2:]]
X_train_kneigboards = train_split[train_split.columns.tolist() [3:]].copy()
X_val_kneigboards= test_split[test_split.columns.tolist() [3:]].copy()
X_test_kneigboards= TEST2[TEST2.columns.tolist() [2:]]
alpha = 0.9
tree1 = GradientBoostingRegressor(loss='quantile', alpha=alpha,
n_estimators=250, max_depth=5,
learning_rate=.1, min_samples_leaf=49,
min_samples_split=49)
 tree1.fit(X_train_kneigboards,Y_train)
 y_upper = tree1.predict(X_val_kneigboards)
 tree1.set_params(alpha=0.1)
 tree1.fit(X_train_kneigboards, Y_train)
 y_lower = tree1.predict(X_val_kneigboards)
 tree1.set_params(loss='ls')
 tree1.fit(X_train_kneigboards, Y_train)
 y_pred = tree1.predict(X_val_kneigboards)
 tree3 = KNeighborsRegressor(n_neighbors=252)
 tree3.fit(X_train_kneigboards,Y_train)
 pred_k = tree3.predict(X_val_kneigboards)
 tree2 = RandomForestRegressor(n_estimators=30,min_samples_leaf=2, min_samples_split=2)
 tree2.fit(X_train,Y_train)
 pred_r = tree2.predict(X_val)
 tree4 = LinearRegression()
tree4.fit(X_train_kneigboards[X_train_kneigboards.columns.tolist() [:-11]],Y_train)
 pred_lr = tree4.predict(X_val_kneigboards[X_val_kneigboards.columns.tolist() [:-11]])
 tree5 = Ridge(alpha=0.03)
 tree5.fit(X_train_kneigboards[X_train_kneigboards.columns.tolist() [:-11]],Y_train)
 pred_ridge = tree5.predict(X_val_kneigboards[X_val_kneigboards.columns.tolist() [:-11]])
 lf = BayesianRidge()
lf.fit(X_train_kneigboards[X_train_kneigboards.columns.tolist() [:]],Y_train)
 pred_baes = lf.predict(X_val_kneigboards[X_val_kneigboards.columns.tolist() [:]])
 print(X_val_kneigboards.columns.tolist() [8:-10])
 print(mean_squared_error(Y_val2,y_upper, squared=False))
print(mean_squared_error(Y_val2,y_lower, squared=False))
print(mean_squared_error(Y_val2,y_pred, squared=False))
print("custom",mean_squared_error(Y_val,X_val2["FVC_PRE"], squared=False))
print("kneugboard",mean_squared_error(Y_val2,pred_k, squared=False))
print("randonf",mean_squared_error(Y_val2,pred_r, squared=False))
print("linear",mean_squared_error(Y_val2,pred_lr, squared=False))
print("ridge",mean_squared_error(Y_val2,pred_ridge, squared=False))
print("baes",mean_squared_error(Y_val2,pred_baes, squared=False))
X_val2["y_upper"] = y_upper
X_val2["y_lower"] = y_lower
X_val2["y_pred"] = y_pred
X_val2["pred_k"] = pred_k
X_val2["pred_r"] = pred_r
X_val2["pred_lr"] = pred_lr
X_val2["pred_ridge"] = pred_ridge
X_val2["baes"] = pred_baes
name = ["baes","pred_r"]
X_val2["Confidence"] = Y_val- X_val2["FVC_PRE"]
X_val2["Confidence"] = X_val2["Confidence"].abs()
delta = 60
X_val2["end"] = X_val2[name].mean(axis=1)
 print("----",X_val2["end"].std())
 X_val2["end"]+=delta
print("mean",mean_squared_error(Y_val,X_val2["end"], squared=False))
test_split["end"]= X_val2["FVC_PRE"]
print(laplace_log_likelihood(Y_val,X_val2.pred_lr,X_val2["Confidence"]))
print(laplace_log_likelihood(Y_val,X_val2.FVC_PRE,X_val2.end))
print(laplace_log_likelihood(Y_val,X_val2.end,X_val2["Confidence"]))
tree1.set_params(loss='ls')
 y_pred2 = tree1.predict(X_test_kneigboards)
 pred_k2 = tree3.predict(X_test_kneigboards)
 pred_r2 = tree2.predict(X_test)
 pred_lr2 = tree4.predict(X_test_kneigboards[X_test_kneigboards.columns.tolist() [:-11]])
 pred_ridge2 = tree5.predict(X_test_kneigboards[X_test_kneigboards.columns.tolist() [:-11]])
 pred_baes2 = lf.predict(X_test_kneigboards[X_test_kneigboards.columns.tolist() [:]])
 TEST2["y_pred"] = y_pred2
TEST2["pred_k"] = pred_k2
TEST2["pred_r"] = pred_r2
TEST2["pred_lr"] = pred_lr2
TEST2["pred_ridge"] = pred_ridge2
TEST2["baes"] = pred_baes2
TEST2["end"] = TEST2[name].mean(axis=1)
 TEST2["end"]+=delta
TEST2['Patient_Week'] = TEST2.agg('{0[Patient]}_{0[week_predict]}'.format, axis=1)
 SUBMISSINO1_pred2 = TEST2[['Patient_Week','FVC_PRE','end',]]
SUBMISSINO1_pred2.columns = ["Patient_Week","FVC","Confidence"]
SUBMISSINO1_pred2.to_csv("submission.csv", index=False)
 SUBMISSINO1_pred2 
<import_modules>
stopWords_en = stopwords.words('english')
 stopWords_ru = stopwords.words('russian')
 tr =new_df_train.copy()
te =new_df_test.copy()
tri=tr.shape[0]
y = tr.deal_probability.copy()
lb=LabelEncoder()
List_Var=['item_id', 'description']
def Concat_Text(df,Columns,Name):
df=df.copy()
df.loc[:,Columns].fillna(" ",inplace=True)
 df[Name]=df[Columns[0]].astype('str')
 for col in Columns[1:]:
df[Name]=df[Name]+' '+df[col].astype('str')
 return df
def Ratio_Words(df):
df=df.copy()
df['description']=df['description'].astype('str')
 df['num_words_description']=df['description'].apply(lambda x:len(x.split()))
Unique_Words=df['description'].apply(lambda x: len(set(x.split())))
 df['Ratio_Words_description']=Unique_Words/df['num_words_description']
return df
def Lenght_Columns(df,Columns):
df=df.copy()
Columns_Len=['len_'+s for s in Columns]
for col in Columns:
df[col]=df[col].astype('str')
 for x,y in zip(Columns,Columns_Len):
df[y]=df[x].apply(len)
 return df
tr_te=tr[tr.columns.difference(["deal_probability"])].append(te)\ 
.pipe(Concat_Text,['city','param_1'],'txt1')\ 
.pipe(Concat_Text,['title','description'],'txt2' ).pipe(Ratio_Words ).pipe(Lenght_Columns,['title','description','param_1'] ).assign(category_name_en=lambda x: pd.Categorical(x['category_name_en'] ).codes,
parent_category_name_en=lambda x:pd.Categorical(x['parent_category_name_en'] ).codes,
region_en=lambda x:pd.Categorical(x['region_en'] ).codes, reg_Time_zone=lambda x:pd.Categorical(x['reg_Time_zone'] ).codes,
user_type=lambda x:pd.Categorical(x['user_type'] ).codes, image_top_1=lambda x:pd.Categorical(x['image_top_1'] ).codes,
param_1=lambda x:lb.fit_transform(x['param_1'].fillna('-1' ).astype('str')) ,
param_2=lambda x:lb.fit_transform(x['param_2'].fillna('-1' ).astype('str')) , param_3=lambda x:lb.fit_transform(x['param_3'].fillna('-1' ).astype('str')) ,
user_id=lambda x:lb.fit_transform(x['user_id'].astype('str')) , reg_dense=lambda x: np.log1p(x['reg_dense'].fillna(0)) ,
city=lambda x:lb.fit_transform(x['city'].astype('str')) , rural=lambda x: np.log1p(x['rural'].fillna(0)) ,
reg_Urban=lambda x: np.log1p(x['reg_Urban'].fillna(0)) ,
price=lambda x: np.log1p(x['price'].fillna(0)) , reg_Population=lambda x: np.log1p(x['reg_Population'].fillna(0)) ,
image_blurrness_score=lambda x: np.log1p(x['image_blurrness_score'].fillna(0)) ,
title=lambda x: x['title'].astype('str')).drop(labels=List_Var,axis=1)
 tr_te.price.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te.reg_Population.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te.reg_Urban.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te.reg_dense.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te.rural.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te.image_blurrness_score.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te
del tr,te
gc.collect()
tr_te.loc[:,'txt2']=tr_te.txt2.apply(lambda x:x.lower().replace("[^[:alpha:]]"," " ).replace("\\s+", " "))
print("Processing Text")
 print("Text 1")
 vec1=CountVectorizer(ngram_range=(1,2),dtype=np.uint8,min_df=5, binary=True,max_features=3000)
 m_tfidf1=vec1.fit_transform(tr_te.txt1)
 tr_te.drop(labels=['txt1'],inplace=True,axis=1)
 print("Text 2")
 vec2=TfidfVectorizer(ngram_range=(1,2),stop_words=stopWords_ru,min_df=3,max_df=0.4,sublinear_tf=True,norm='l2',max_features=5500,dtype=np.uint8)
 m_tfidf2=vec2.fit_transform(tr_te.txt2)
 tr_te.drop(labels=['txt2'],inplace=True,axis=1)
 print("Title")
 vec3=CountVectorizer(ngram_range=(3,6),analyzer='char_wb',dtype=np.uint8,min_df=5, binary=True,max_features=2000)
 m_tfidf3=vec3.fit_transform(tr_te.title)
 tr_te.drop(labels=['title'],inplace=True,axis=1)
 data = hstack(( tr_te.values,m_tfidf1,m_tfidf2,m_tfidf3)).tocsr()
print(data.shape)
 del tr_te,m_tfidf1,m_tfidf2,m_tfidf3
gc.collect()
dtest=data[tri:]
X=data[:tri]
del data
gc.collect()
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.15, random_state=23)
import numpy as np
import pandas as pd
import os
from tqdm import tqdm, trange
from sklearn.metrics import roc_auc_score
import pickle
import gc<define_variables>
dtrain =lgb.Dataset(data = X_train, label = y_train)
 dval =lgb.Dataset(data = X_valid, label = y_valid)
 Dparam = {'objective' : 'regression',
'boosting_type': 'gbdt',
'metric' : 'rmse',
'nthread' : 4,
'shrinkage_rate':0.02,
'max_depth':18,
'min_child_weight': 8,
'bagging_fraction':0.75,
'feature_fraction':0.75,
'lambda_l1':0,
'lambda_l2':0,
'num_leaves':31,
'verbosity' : -1}
print("Training Model")
 m_lgb=lgb.train(params=Dparam,train_set=dtrain,num_boost_round=40000, early_stopping_rounds=500, valid_sets=[dtrain,dval], verbose_eval=50,valid_names=['train','valid'])
 fig, ax = plt.subplots(figsize=(10, 14))
lgb.plot_importance(m_lgb, max_num_features=50, ax=ax)
 plt.title("Light GBM Feature Importance")
 Pred=m_lgb.predict(dtest)
 Pred[Pred<0]=0
Pred[Pred>1]=1
print("Output Model")
 LGB_text=pd.read_csv(".. /input/avito-demand-prediction/sample_submission.csv")
 LGB_text['deal_probability']=Pred
LGB_text.to_csv("no_Actdate.csv", index=False )