Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
train_df = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test_df = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 X_train = train_df.drop(['SalePrice'], axis=1)
 y_train = train_df['SalePrice']
train_df
combined = pd.concat([train_df,test_df])
 all_data = [train_df,test_df]
total = combined.isnull().sum().sort_values(ascending=False)
 percent_1 = combined.isnull().sum() /combined.isnull().count() *100
percent_2 =(round(percent_1, 1)).sort_values(ascending=False)
 missing_data = pd.concat([total, percent_2], axis=1, keys=['Total', '%'])
 missing_data.head(20 )
df1 = pd.read_csv('.. /input/fork-of-2heads-looper-super-puper/submission.csv')
 df2 = pd.read_csv('.. /input/pytorch-transfer-learning-with-k-folds-by-drug-ids/submission.csv')
 df3 = pd.read_csv('.. /input/blend-moa-pytorch-rankgauss-pca-nn-2loss/submission.csv' )<sort_values>
combined.groupby('MSZoning')['Id'].count()
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )<sort_values>
combined[combined['MSZoning'].isnull() ]
df1 = order_sub(df1)
 df2 = order_sub(df2)
 df3 = order_sub(df3 )<prepare_output>
MSZoningMap = {'FV': 1, 'RL': 2, 'RH': 3, 'RM': 4, 'C(all)': 5}
for d in all_data:
d['MSZoning'] = d['MSZoning'].fillna('RL')
 d['MSZoning'] = d['MSZoning'].map(MSZoningMap)
 combined['MSZoning'] = combined['MSZoning'].fillna('RL')
 combined['MSZoning'] = combined['MSZoning'].map(MSZoningMap)
 combined['MSZoning'].unique()
BLEND=df1.copy()
BLEND.iloc[:,1:] =(df1.iloc[:,1:]+df2.iloc[:,1:]+df3.iloc[:,1:])/3.0<define_variables>
subclass_map = train_df.groupby('MSSubClass')['SalePrice'].mean().sort_values()
for d in all_data:
for idx, s in enumerate(subclass_map.index):
d.loc[d['MSSubClass'] == s,'MSSubClass'] = idx+1
for idx, s in enumerate(subclass_map.index):
combined.loc[combined['MSSubClass'] == s,'MSSubClass'] = idx+1
train_df.groupby('MSSubClass')['SalePrice'].describe().sort_values('mean' )
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
NUM_TARGET = len(TARGET_COL )<load_from_csv>
combined['MSSubClass'].isnull().sum()
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 df = pd.read_csv("/kaggle/input/lish-moa/sample_submission.csv")
 public_id = list(df['sig_id'].values)
 df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )<set_options>
combined['LotFrontage'] = combined['LotFrontage'].fillna(0)
 for d in all_data:
d['LotFrontage'] = d['LotFrontage'].fillna(0)
 d['LotFrontage'] =(d['LotFrontage']-combined['LotFrontage'].mean())/combined['LotFrontage'].std()
combined['LotFrontage'] =(combined['LotFrontage']-combined['LotFrontage'].mean())/combined['LotFrontage'].std()
warnings.filterwarnings('ignore' )<import_modules>
plt.plot(y_train, X_train['LotArea'], 'o')
 m, b = np.polyfit(y_train, X_train['LotArea'], 1)
 plt.plot(y_train, m*y_train + b )
from sklearn.preprocessing import QuantileTransformer<load_from_csv>
combined['LotArea'] = combined['LotArea'].fillna(0)
 for d in all_data:
d['LotArea'] = d['LotArea'].fillna(0)
 d['LotArea'] =(d['LotArea']-combined['LotArea'].mean())/combined['LotArea'].std()
combined['LotArea'] =(combined['LotArea']-combined['LotArea'].mean())/combined['LotArea'].std()
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 df = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<create_dataframe>
combined['Alley'] = combined['Alley'].fillna('None')
 for d in all_data:
d['Alley'] = d['Alley'].fillna('None' )
train_features2=train_features.copy()
test_features2=test_features.copy()<define_variables>
combined.drop(['Street','Alley'], axis=1, inplace=True)
 for d in all_data:
d.drop(['Street','Alley'], axis=1, inplace=True )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<categorify>
combined.drop(['LotShape','LandContour', 'Utilities', 'LandSlope', \
'Condition1', 'Condition2', 'YearBuilt', 'RoofStyle', \
'RoofMatl', 'Heating', 'Electrical', 'Functional', \
'GarageYrBlt', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', \
'PoolArea', 'PoolQC', 'Fence', 'MiscFeature', 'MiscVal', \
'SaleType'], axis=1, inplace=True)
 for d in all_data:
d.drop(['LotShape','LandContour', 'Utilities', 'LandSlope', \
'Condition1', 'Condition2', 'YearBuilt', 'RoofStyle', \
'RoofMatl', 'Heating', 'Electrical', 'Functional', \
'GarageYrBlt', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', \
'PoolArea', 'PoolQC', 'Fence', 'MiscFeature', 'MiscVal', \
'SaleType'], axis=1, inplace=True )
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
subclass_map = train_df.groupby('LotConfig')['SalePrice'].mean().sort_values()
for d in all_data:
for idx, s in enumerate(subclass_map.index):
d.loc[d['LotConfig'] == s,'LotConfig'] = idx+1
for idx, s in enumerate(subclass_map.index):
combined.loc[combined['LotConfig'] == s,'LotConfig'] = idx+1
train_df.groupby('LotConfig')['SalePrice'].describe().sort_values('mean' )
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<create_dataframe>
combined['NeighborhoodLotArea'] = combined['Neighborhood'] * combined['LotArea']
for d in all_data:
d['NeighborhoodLotArea'] = d['Neighborhood'] * d['LotArea']
d['NeighborhoodLotArea'] =(d['NeighborhoodLotArea'] - combined['NeighborhoodLotArea'].min())\
/(combined['NeighborhoodLotArea'].max() -combined['NeighborhoodLotArea'].min())
 combined['NeighborhoodLotArea'] =(combined['NeighborhoodLotArea'] - combined['NeighborhoodLotArea'].min())\
/(combined['NeighborhoodLotArea'].max() -combined['NeighborhoodLotArea'].min() )
n_comp = 600
pca_g = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 gpca=(pca_g.fit(data[GENES]))
train2=(gpca.transform(train_features[GENES]))
test2 =(gpca.transform(test_features[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
 dump(gpca, open('gpca.pkl', 'wb'))<create_dataframe>
subclass_map = train_df.groupby('BldgType')['SalePrice'].mean().sort_values()
for d in all_data:
for idx, s in enumerate(subclass_map.index):
d.loc[d['BldgType'] == s,'BldgType'] = idx+1
for idx, s in enumerate(subclass_map.index):
combined.loc[combined['BldgType'] == s,'BldgType'] = idx+1
train_df.groupby('BldgType')['SalePrice'].describe().sort_values('mean' )
n_comp = 50
pca_c = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 cpca=(pca_c.fit(data[CELLS]))
train2=(cpca.transform(train_features[CELLS]))
test2 =(cpca.transform(test_features[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1)
 dump(cpca, open('cpca.pkl', 'wb'))<concatenate>
combined['NeighborhoodBldgType'] = combined['Neighborhood'] * combined['BldgType']
for d in all_data:
d['NeighborhoodBldgType'] = d['Neighborhood'] * d['BldgType']
d['NeighborhoodBldgType'] =(d['NeighborhoodBldgType'] - combined['NeighborhoodBldgType'].min())\
/(combined['NeighborhoodBldgType'].max() -combined['NeighborhoodBldgType'].min())
 combined['NeighborhoodBldgType'] =(combined['NeighborhoodBldgType'] - combined['NeighborhoodBldgType'].min())\
/(combined['NeighborhoodBldgType'].max() -combined['NeighborhoodBldgType'].min() )
c_n = [f for f in list(train_features.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_features[c_n].var() >= 0.85 ).values
tmp = train_features[c_n].loc[:, mask]
train_features = pd.concat([train_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_features[c_n].loc[:, mask]
test_features = pd.concat([test_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )<define_search_model>
combined['TotalBsmtSF'] = combined['TotalBsmtSF'].fillna(0)
 combined['GarageArea'] = combined['GarageArea'].fillna(0)
 combined['TotalSquareFeet'] = combined['TotalBsmtSF'] + combined['1stFlrSF'] \
+ combined['2ndFlrSF'] + combined['GarageArea'] \
+ combined['WoodDeckSF']
for d in all_data:
d['TotalBsmtSF'] = d['TotalBsmtSF'].fillna(0)
 d['GarageArea'] = d['GarageArea'].fillna(0)
 d['TotalSquareFeet'] = d['TotalBsmtSF'] + d['1stFlrSF'] + d['2ndFlrSF'] + d['GarageArea'] + d['WoodDeckSF']
d['TotalSquareFeet'] =(d['TotalSquareFeet']-combined['TotalSquareFeet'].mean())/combined['TotalSquareFeet'].std()
combined['TotalSquareFeet'] =(combined['TotalSquareFeet']-combined['TotalSquareFeet'].mean())/combined['TotalSquareFeet'].std()
def fe_cluster_genes(train, test, n_clusters_g = 22, SEED = 42):
features_g = GENES
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_genes = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_genes, open('kmeans_genes.pkl', 'wb'))
train[f'clusters_{kind}'] = kmeans_genes.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_genes.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 return train, test
train_features2 ,test_features2=fe_cluster_genes(train_features2,test_features2 )<find_best_model_class>
subclass_map = train_df.groupby('HouseStyle')['SalePrice'].mean().sort_values()
for d in all_data:
for idx, s in enumerate(subclass_map.index):
d.loc[d['HouseStyle'] == s,'HouseStyle'] = idx+1
for idx, s in enumerate(subclass_map.index):
combined.loc[combined['HouseStyle'] == s,'HouseStyle'] = idx+1
train_df.groupby('HouseStyle')['SalePrice'].describe().sort_values('mean' )
def fe_cluster_cells(train, test, n_clusters_c = 4, SEED = 42):
features_c = CELLS
def create_cluster(train, test, features, kind = 'c', n_clusters = n_clusters_c):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_cells = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_cells, open('kmeans_cells.pkl', 'wb'))
train[f'clusters_{kind}'] = kmeans_cells.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_cells.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster_cells(train_features2,test_features2 )<concatenate>
combined['NeighborhoodHouseStyle'] = combined['Neighborhood'] * combined['HouseStyle']
for d in all_data:
d['NeighborhoodHouseStyle'] = d['Neighborhood'] * d['HouseStyle']
d['NeighborhoodHouseStyle'] =(d['NeighborhoodHouseStyle'] - combined['NeighborhoodHouseStyle'].min())\
/(combined['NeighborhoodHouseStyle'].max() -combined['NeighborhoodHouseStyle'].min())
 combined['NeighborhoodHouseStyle'] =(combined['NeighborhoodHouseStyle'] - combined['NeighborhoodHouseStyle'].min())\
/(combined['NeighborhoodHouseStyle'].max() -combined['NeighborhoodHouseStyle'].min() )
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )<categorify>
combined['OverallQualNCond'] = combined['OverallQual'] * combined['OverallCond']
for d in all_data:
d['OverallQualNCond'] = d['OverallQual'] * d['OverallCond']
d['OverallQualNCond'] =(d['OverallQualNCond'] - combined['OverallQualNCond'].min())\
/(combined['OverallQualNCond'].max() -combined['OverallQualNCond'].min())
 combined['OverallQualNCond'] =(combined['OverallQualNCond'] - combined['OverallQualNCond'].min())\
/(combined['OverallQualNCond'].max() -combined['OverallQualNCond'].min() )
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans_pca = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_pca, open('kmeans_pca.pkl', 'wb'))
train[f'clusters_pca'] = kmeans_pca.predict(train.values)
 test[f'clusters_pca'] = kmeans_pca.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )<split>
combined['NeighborhoodOverallQualNCond'] = combined['Neighborhood'] * combined['OverallQualNCond']
for d in all_data:
d['NeighborhoodOverallQualNCond'] = d['Neighborhood'] * d['OverallQualNCond']
d['NeighborhoodOverallQualNCond'] =(d['NeighborhoodOverallQualNCond'] - combined['NeighborhoodOverallQualNCond'].min())\
/(combined['NeighborhoodOverallQualNCond'].max() -combined['NeighborhoodOverallQualNCond'].min())
 combined['NeighborhoodOverallQualNCond'] =(combined['NeighborhoodOverallQualNCond'] - combined['NeighborhoodOverallQualNCond'].min())\
/(combined['NeighborhoodOverallQualNCond'].max() -combined['NeighborhoodOverallQualNCond'].min() )
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]<split>
max_year = combined['YrSold'].max()
max_month = combined.loc[combined['YrSold'] == max_year, 'MoSold'].max()
for d in all_data:
months_ago =(max_year-d['YearRemodAdd'])*12 + max_month
d['AdjRemodAdd'] =(months_ago - months_ago.min())/(months_ago.max() -months_ago.min())\
* -1 + 1
d.drop(['YearRemodAdd'], axis=1, inplace=True)
 months_ago =(max_year-combined['YearRemodAdd'])*12 + max_month
combined['AdjRemodAdd'] =(months_ago - months_ago.min())/(months_ago.max() -months_ago.min())\
* -1 + 1
combined.drop(['YearRemodAdd'], axis=1, inplace=True )
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]<define_variables>
df = train_df.groupby(['YrSold','MoSold'])[['SalePrice']].mean()
df = df.rename({'SalePrice': 'MonthMean'}, axis=1)
 qt =(df[['MonthMean']] + df[['MonthMean']].shift(1)+ df[['MonthMean']].shift(-1)) / 3
df['QuarterMean'] = qt['MonthMean']
df['QuarterMean'] = df['QuarterMean'].fillna(df['MonthMean'])
 print(df )
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']<feature_engineering>
for idx, d in enumerate(all_data):
all_data[idx] = pd.merge(d, df, how='left', left_on=['YrSold','MoSold'], right_on=['YrSold','MoSold'])
 all_data[idx].drop(['MoSold', 'YrSold'], axis=1, inplace=True)
 combined = pd.merge(combined, df, how='left', left_on=['YrSold','MoSold'], right_on=['YrSold','MoSold'])
 combined.drop(['MoSold', 'YrSold'], axis=1, inplace=True)
 train_df = all_data[0]
test_df = all_data[1]
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )<split>
for d in all_data:
d['QuarterMeanNorm'] =(d['QuarterMean']-combined['QuarterMean'].mean())/combined['QuarterMean'].std()
d['MonthMeanNorm'] =(d['MonthMean']-combined['MonthMean'].mean())/combined['MonthMean'].std()
combined['QuarterMeanNorm'] =(combined['QuarterMean']-combined['QuarterMean'].mean())/combined['QuarterMean'].std()
combined['MonthMeanNorm'] =(combined['MonthMean']-combined['MonthMean'].mean())/combined['MonthMean'].std()
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]<concatenate>
ex1_map = {'Other': 0, 'BrkFace': 1, 'CemntBd': 2, 'Plywood': 3, 'Wd Sdng': 4, 'MetalSd': 5, 'HdBoard': 6, 'VinylSd': 7}
ex2_map = {'Other': 0, 'AsbShng': 1, 'BrkFace': 2, 'Stucco': 3, 'Wd Shng': 4, 'CmentBd': 5, 'Plywood': 7, 'Wd Sdng': 8, 'HdBoard': 9, 'MetalSd': 10, 'VinylSd': 11}
max_map = {'None': 0, 'BrkCmn': 1, 'Stone': 2, 'BrkFace': 3}
exg_map = {'Po': 0.0, 'Fa': 0.25, 'TA': 0.5, 'Gd': 0.75, 'Ex': 1.0}
combined['MasVnrArea'] = combined['MasVnrArea'].fillna(0)
 for d in all_data:
d['Exterior1st'] = d['Exterior1st'].fillna(combined['Exterior1st'].mode() [0])
 d['Exterior1st'] = d['Exterior1st'].replace(['AsphShn','CBlock','ImStucc','BrkComm','Stone','AsbShng','Stucco','WdShing'], 'Other')
 d['Exterior1st'] = d['Exterior1st'].map(ex1_map)
 d['Exterior2nd'] = d['Exterior2nd'].fillna(combined['Exterior2nd'].mode() [0])
 d['Exterior2nd'] = d['Exterior2nd'].replace(['CBlock','AsphShn','Stone','Brk Cmn','ImStucc'], 'Other')
 d['Exterior2nd'] = d['Exterior2nd'].map(ex2_map)
 d['MasVnrType'] = d['MasVnrType'].fillna('None')
 d['MasVnrType'] = d['MasVnrType'].map(max_map)
 d['MasVnrArea'] = d['MasVnrArea'].fillna(0)
 d['MasVnrArea'] =(d['MasVnrArea'] - combined['MasVnrArea'].min())\
/(combined['MasVnrArea'].max() -combined['MasVnrArea'].min())
 d['ExterQual'] = d['ExterQual'].map(exg_map)
 d['ExterCond'] = d['ExterCond'].map(exg_map)
 combined['Exterior1st'] = combined['Exterior1st'].fillna(combined['Exterior1st'].mode() [0])
 combined['Exterior1st'] = combined['Exterior1st'].replace(['AsphShn','CBlock','ImStucc','BrkComm','Stone','AsbShng','Stucco','WdShing'], 'Other')
 combined['Exterior1st'] = combined['Exterior1st'].map(ex1_map)
 combined['Exterior2nd'] = combined['Exterior2nd'].fillna(combined['Exterior2nd'].mode() [0])
 combined['Exterior2nd'] = combined['Exterior2nd'].replace(['CBlock','AsphShn','Stone','Brk Cmn','ImStucc'], 'Other')
 combined['Exterior2nd'] = combined['Exterior2nd'].map(ex2_map)
 combined['MasVnrType'] = combined['MasVnrType'].fillna('None')
 combined['MasVnrType'] = combined['MasVnrType'].map(max_map)
 combined['MasVnrArea'] =(combined['MasVnrArea'] - combined['MasVnrArea'].min())\
/(combined['MasVnrArea'].max() -combined['MasVnrArea'].min())
 combined['ExterQual'] = combined['ExterQual'].map(exg_map)
 combined['ExterCond'] = combined['ExterCond'].map(exg_map )
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )<merge>
fnd_map = {'Slab': 0, 'BrkTil': 1, 'CBlock': 2, 'Other': 3, 'PConc': 4}
bsm_map = {'Po': 0.0, 'Fa': 0.25, 'TA': 0.5, 'Gd': 0.75, 'Ex': 1.0}
bex_map = {'NA': 0.0, 'No': 0.25, 'Mn': 0.5, 'Av': 0.75, 'Gd': 1.0}
bfi_map = {'NA': 0.0, 'Unf': 0.17, 'LwQ': 0.33, 'Rec': 0.5, 'BLQ': 0.67, 'ALQ': 0.83, 'GLQ': 1.0}
combined['BsmtFinSF1'] = combined['BsmtFinSF1'].fillna(0)
 combined['BsmtFinSF2'] = combined['BsmtFinSF2'].fillna(0)
 combined['BsmtUnfSF'] = combined['BsmtUnfSF'].fillna(0)
 combined['TotalBsmtSF'] = combined['TotalBsmtSF'].fillna(0)
 combined['BsmtFullBath'] = combined['BsmtFullBath'].fillna(0)
 combined['BsmtHalfBath'] = combined['BsmtFullBath'].fillna(0)
 for d in all_data:
d['Foundation'] = d['Foundation'].replace(['Stone', 'Wood'], 'Other')
 d['Foundation'] = d['Foundation'].map(fnd_map)
 d['BsmtQual'] = d['BsmtQual'].fillna('Po')
 d['BsmtQual'] = d['BsmtQual'].map(bsm_map)
 d['BsmtCond'] = d['BsmtCond'].fillna('Po')
 d['BsmtCond'] = d['BsmtCond'].map(bsm_map)
 d['BsmtExposure'] = d['BsmtExposure'].fillna('NA')
 d['BsmtExposure'] = d['BsmtExposure'].map(bex_map)
 d['BsmtFinType1'] = d['BsmtFinType1'].fillna('NA')
 d['BsmtFinType1'] = d['BsmtFinType1'].map(bfi_map)
 d['BsmtFinType2'] = d['BsmtFinType2'].fillna('NA')
 d['BsmtFinType2'] = d['BsmtFinType2'].map(bfi_map)
 d['BsmtFinSF1'] = d['BsmtFinSF1'].fillna(0)
 d['BsmtFinSF1'] =(d['BsmtFinSF1']-combined['BsmtFinSF1'].mean())/combined['BsmtFinSF1'].std()
d['BsmtFinSF2'] = d['BsmtFinSF2'].fillna(0)
 d['BsmtFinSF2'] =(d['BsmtFinSF2']-combined['BsmtFinSF2'].mean())/combined['BsmtFinSF2'].std()
d['BsmtUnfSF'] = d['BsmtUnfSF'].fillna(0)
 d['BsmtUnfSF'] =(d['BsmtUnfSF']-combined['BsmtUnfSF'].mean())/combined['BsmtUnfSF'].std()
d['TotalBsmtSF'] = d['TotalBsmtSF'].fillna(0)
 d['TotalBsmtSF'] =(d['TotalBsmtSF']-combined['TotalBsmtSF'].mean())/combined['TotalBsmtSF'].std()
d['BsmtFullBath'] = d['BsmtFullBath'].fillna(0)
 d['BsmtHalfBath'] = d['BsmtFullBath'].fillna(0)
 d['BsmtFullBath'] =(d['BsmtFullBath']-combined['BsmtFullBath'].mean())/combined['BsmtFullBath'].std()
d['BsmtHalfBath'] =(d['BsmtHalfBath']-combined['BsmtHalfBath'].mean())/combined['BsmtHalfBath'].std()
combined['Foundation'] = combined['Foundation'].replace(['Stone', 'Wood'], 'Other')
 combined['Foundation'] = combined['Foundation'].map(fnd_map)
 combined['BsmtQual'] = combined['BsmtQual'].fillna('Po')
 combined['BsmtQual'] = combined['BsmtQual'].map(bsm_map)
 combined['BsmtCond'] = combined['BsmtCond'].fillna('Po')
 combined['BsmtCond'] = combined['BsmtCond'].map(bsm_map)
 combined['BsmtExposure'] = combined['BsmtExposure'].fillna('NA')
 combined['BsmtExposure'] = combined['BsmtExposure'].map(bex_map)
 combined['BsmtFinType1'] = combined['BsmtFinType1'].fillna('NA')
 combined['BsmtFinType1'] = combined['BsmtFinType1'].map(bfi_map)
 combined['BsmtFinType2'] = combined['BsmtFinType2'].fillna('NA')
 combined['BsmtFinType2'] = combined['BsmtFinType2'].map(bfi_map)
 combined['BsmtFinSF1'] =(combined['BsmtFinSF1']-combined['BsmtFinSF1'].mean())/combined['BsmtFinSF1'].std()
combined['BsmtFinSF2'] =(combined['BsmtFinSF2']-combined['BsmtFinSF2'].mean())/combined['BsmtFinSF2'].std()
combined['BsmtUnfSF'] =(combined['BsmtUnfSF']-combined['BsmtUnfSF'].mean())/combined['BsmtUnfSF'].std()
combined['TotalBsmtSF'] =(combined['TotalBsmtSF']-combined['TotalBsmtSF'].mean())/combined['TotalBsmtSF'].std()
combined['BsmtFullBath'] =(combined['BsmtFullBath']-combined['BsmtFullBath'].mean())/combined['BsmtFullBath'].std()
combined['BsmtHalfBath'] =(combined['BsmtHalfBath']-combined['BsmtHalfBath'].mean())/combined['BsmtHalfBath'].std()
train = train_features.merge(train_targets_nonscored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_nonscored.columns]<drop_column>
ht_map = {'Po': 0.0, 'Fa': 0.25, 'TA': 0.5, 'Gd': 0.75, 'Ex': 1.0}
yn_map = {'N': 0, 'Y': 1}
for d in all_data:
d['HeatingQC'] = d['HeatingQC'].map(ht_map)
 d['CentralAir'] = d['CentralAir'].map(yn_map)
 combined['HeatingQC'] = combined['HeatingQC'].map(ht_map)
 combined['CentralAir'] = combined['CentralAir'].map(yn_map )
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
grt_map = {'None': 0, 'Detchd': 1, 'Other': 2, 'Attchd': 3, 'BuiltIn': 4}
grf_map = {'None': 0, 'Unf': 1, 'RFn': 2, 'Fin': 3}
gqc_map = {'Po': 0.0, 'Fa': 0.25, 'TA': 0.5, 'Gd': 0.75, 'Ex': 1.0}
pdr_map = {'N': 0.0, 'P': 0.5, 'Y': 1.0}
combined['GarageCars'] = combined['GarageCars'].fillna(0)
 combined['GarageArea'] = combined['GarageArea'].fillna(0)
 for d in all_data:
d['GarageType'] = d['GarageType'].fillna('None')
 d['GarageType'] = d['GarageType'].replace(['CarPort', '2Types', 'Basment'], 'Other')
 d['GarageType'] = d['GarageType'].map(grt_map)
 d['GarageFinish'] = d['GarageFinish'].fillna('None')
 d['GarageFinish'] = d['GarageFinish'].map(grf_map)
 d['GarageCars'] = d['GarageCars'].fillna(0)
 d['GarageCars'] =(d['GarageCars']-combined['GarageCars'].mean())/combined['GarageCars'].std()
d['GarageArea'] = d['GarageArea'].fillna(0)
 d['GarageArea'] =(d['GarageArea']-combined['GarageArea'].mean())/combined['GarageArea'].std()
d['GarageQual'] = d['GarageQual'].fillna('Po')
 d['GarageQual'] = d['GarageQual'].map(gqc_map)
 d['GarageCond'] = d['GarageCond'].fillna('Po')
 d['GarageCond'] = d['GarageCond'].map(gqc_map)
 d['PavedDrive'] = d['PavedDrive'].map(pdr_map)
 combined['GarageType'] = combined['GarageType'].fillna('None')
 combined['GarageType'] = combined['GarageType'].replace(['CarPort', '2Types', 'Basment'], 'Other')
 combined['GarageType'] = combined['GarageType'].map(grt_map)
 combined['GarageFinish'] = combined['GarageFinish'].fillna('None')
 combined['GarageFinish'] = combined['GarageFinish'].map(grf_map)
 combined['GarageCars'] =(combined['GarageCars']-combined['GarageCars'].mean())/combined['GarageCars'].std()
combined['GarageArea'] =(combined['GarageArea']-combined['GarageArea'].mean())/combined['GarageArea'].std()
combined['GarageQual'] = combined['GarageQual'].fillna('Po')
 combined['GarageQual'] = combined['GarageQual'].map(gqc_map)
 combined['GarageCond'] = combined['GarageCond'].fillna('Po')
 combined['GarageCond'] = combined['GarageCond'].map(gqc_map)
 combined['PavedDrive'] = combined['PavedDrive'].map(pdr_map )
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<categorify>
for d in all_data:
d['WoodDeckSF'] =(d['WoodDeckSF']-combined['WoodDeckSF'].mean())/combined['WoodDeckSF'].std()
d['OpenPorchSF'] =(d['OpenPorchSF']-combined['OpenPorchSF'].mean())/combined['OpenPorchSF'].std()
combined['WoodDeckSF'] =(combined['WoodDeckSF']-combined['WoodDeckSF'].mean())/combined['WoodDeckSF'].std()
combined['OpenPorchSF'] =(combined['OpenPorchSF']-combined['OpenPorchSF'].mean())/combined['OpenPorchSF'].std()
train = pd.get_dummies(train, columns=['cp_time','cp_dose'])
 test_ = pd.get_dummies(test, columns=['cp_time','cp_dose'] )<define_variables>
sac_map = {'Other': 0, 'Abnorml': 1, 'Normal': 2, 'Partial': 3}
for d in all_data:
d['SaleCondition'] = d['SaleCondition'].replace(['AdjLand', 'Family', 'Alloca'], 'Other')
 d['SaleCondition'] = d['SaleCondition'].map(sac_map)
 combined['SaleCondition'] = combined['SaleCondition'].replace(['AdjLand', 'Family', 'Alloca'], 'Other')
 combined['SaleCondition'] = combined['SaleCondition'].map(sac_map )
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]<categorify>
pd.set_option('display.max_columns', None)
 train_df.head(10 )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct 
<train_model>
train_df.to_csv('new_train.csv', index=False)
 test_df.to_csv('new_test.csv', index=False)
 print("New files have been created." )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<normalization>
X = pd.read_csv("/kaggle/working/new_train.csv")
 y = pd.read_csv("/kaggle/working/new_test.csv")
 all_data = [X,y]
pd.set_option('display.max_columns', None)
 X.head(20 )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<normalization>
B = pd.concat([X,y])
 B = pd.concat([B, pd.get_dummies(B['Neighborhood'], prefix='Neighborhood', drop_first=True)], axis=1)
 B = pd.concat([B, pd.get_dummies(B['LotConfig'], prefix='LotConfig', drop_first=True)], axis=1)
 B = pd.concat([B, pd.get_dummies(B['BldgType'], prefix='BldgType', drop_first=True)], axis=1)
 B = pd.concat([B, pd.get_dummies(B['HouseStyle'], prefix='HouseStyle', drop_first=True)], axis=1)
 B = pd.concat([B, pd.get_dummies(B['SaleCondition'], prefix='SaleCondition', drop_first=True)], axis=1)
 B = pd.concat([B, pd.get_dummies(B['GarageFinish'], prefix='GarageFinish', drop_first=True)], axis=1)
 B = pd.concat([B, pd.get_dummies(B['GarageType'], prefix='GarageType', drop_first=True)], axis=1)
 B = pd.concat([B, pd.get_dummies(B['Foundation'], prefix='Foundation', drop_first=True)], axis=1)
 B.drop(['LotConfig', 'BldgType', 'HouseStyle', 'SaleCondition', 'GarageFinish', 'GarageType', 'Foundation'], axis=1, inplace=True)
 B.drop(['MasVnrType', 'Exterior2nd', 'Exterior1st', 'MSZoning', 'MSSubClass'], axis=1, inplace=True)
 B.head(20 )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.leaky_relu(self.dense1(x), 1e-3)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<define_variables>
X = B[:len(X)]
y = B[len(X):]
y = y.drop('SalePrice', axis=1)
 len(X )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048<split>
X.to_csv('new_train_1h.csv', index=False)
 y.to_csv('new_test_1h.csv', index=False)
 print("New files have been created." )
def run_training(fold, seed):
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7,random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
train.loc[v_idx, 'kfold'] = int(f)
 train['kfold'] = train['kfold'].astype(int)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"SEED{seed}_FOLD{fold}_nonscored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"SEED{seed}_FOLD{fold}_nonscored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions 
<categorify>
X = pd.read_csv("/kaggle/working/new_train_1h.csv")
 test_df = pd.read_csv("/kaggle/working/new_test_1h.csv" )
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<define_variables>
train_df = X.sample(frac=0.6,random_state=200)
 evaluate_df = X.drop(train_df.index)
 X_train = train_df.drop(['Id','SalePrice'], axis=1)
 Y_train = train_df["SalePrice"]
X_train_full = X.drop(['Id','SalePrice'], axis=1)
 Y_train_full = X["SalePrice"]
X_eval = evaluate_df.drop(['Id','SalePrice'], axis=1)
 Y_eval = evaluate_df["SalePrice"]
X_test = test_df.drop('Id', axis=1)
 len(X_train_full.columns )
SEED = [0,1,2,3,4,5,6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test_[target_cols] = predictions<merge>
grad_boost = xgb.XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=5, min_child_weight=1.5, n_estimators=1900,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 grad_boost_full = xgb.XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=5, min_child_weight=1.5, n_estimators=1900,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 num_feat_to_drop = 1
dropped_features = []
i = 0
best_acc = 1
while len(X_train.columns)> 5:
grad_boost.fit(X_train, Y_train)
 grad_boost_full.fit(X_train_full, Y_train_full)
 acc_train = np.sqrt(mean_squared_log_error(Y_train, grad_boost.predict(X_train)))
 acc_eval = np.sqrt(mean_squared_log_error(Y_eval, grad_boost.predict(X_eval)))
 acc_full = np.sqrt(mean_squared_log_error(Y_train_full, grad_boost_full.predict(X_train_full)))
 avg_acc =(acc_train + acc_eval + acc_full)/ 3
if(avg_acc >= best_acc):
print(f'{i}: Train = {acc_train} Evaluate = {acc_eval} Full = {acc_full} Average = {avg_acc}')
 else:
best_acc = avg_acc
print(f'{i}: Train = {acc_train} Evaluate = {acc_eval} Full = {acc_full} Average = {avg_acc} - NEW BEST!!!')
 print(f' Dropped: {dropped_features}')
 importances = pd.DataFrame({'feature':X_train.columns,'importance':np.round(grad_boost.feature_importances_,3)})
 importances = importances.sort_values('importance',ascending=True ).set_index('feature')
 features_to_drop = importances[0:num_feat_to_drop].index.tolist()
if len(features_to_drop)<= 0:
break
X_train.drop(features_to_drop, axis=1, inplace=True)
 X_eval.drop(features_to_drop, axis=1, inplace=True)
 X_train_full.drop(features_to_drop, axis=1, inplace=True)
 dropped_features.append(features_to_drop)
 i += num_feat_to_drop
train = train.merge(train_targets_scored, on='sig_id')
 target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<define_variables>
best_dropped = [['LotConfig_4'], ['Foundation_3'], ['Neighborhood_3'], ['BldgType_2'], ['Neighborhood_21'], ['Neighborhood_12'], ['Neighborhood_8'], ['HouseStyle_8'], ['BldgType_3'], ['HouseStyle_2'], ['Neighborhood_10']]
best_dropped = [x for inner in best_dropped for x in inner]
best_dropped
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id','kfold']]<define_variables>
X = pd.read_csv("/kaggle/working/new_train_1h.csv")
 test_df = pd.read_csv("/kaggle/working/new_test_1h.csv")
 train_df = X.sample(frac=0.6,random_state=200)
 evaluate_df = X.drop(train_df.index)
 X_train = train_df.drop(['Id','SalePrice'], axis=1)
 Y_train = train_df["SalePrice"]
X_train_full = X.drop(['Id','SalePrice'], axis=1)
 Y_train_full = X["SalePrice"]
X_eval = evaluate_df.drop(['Id','SalePrice'], axis=1)
 Y_eval = evaluate_df["SalePrice"]
X_test = test_df.drop('Id', axis=1 )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048<split>
for(columnName, columnData)in X_test.iteritems() :
print(f'{columnName}: {X_test[columnName].isna().sum() }' )
def run_training(fold, seed):
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7,random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
train.loc[v_idx, 'kfold'] = int(f)
 train['kfold'] = train['kfold'].astype(int)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"SEED{seed}_FOLD{fold}_scored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"SEED{seed}_FOLD{fold}_scored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions 
<categorify>
drop_list = ['LotConfig_4', 'Foundation_3', 'Neighborhood_3', 'BldgType_2', 'Neighborhood_21', 'Neighborhood_12', 'Neighborhood_8', 'HouseStyle_8', 'BldgType_3', 'HouseStyle_2', 'Neighborhood_10']
X_train.drop(drop_list, axis=1,inplace=True)
 X_eval.drop(drop_list, axis=1,inplace=True)
 X_train_full.drop(drop_list, axis=1,inplace=True)
 X_test.drop(drop_list, axis=1,inplace=True )
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<define_variables>
lr = linear_model.LinearRegression()
lr.fit(X_train, Y_train)
 acc_train_lr = np.sqrt(mean_squared_log_error(Y_train, lr.predict(X_train)))
 acc_eval_lr = np.sqrt(mean_squared_log_error(Y_eval, lr.predict(X_eval)))
 print(f'LR: Train = {acc_train_lr} Evaluate = {acc_eval_lr}' )
SEED = [0,1,2,3,4,5,6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions<compute_train_metric>
sgd = linear_model.SGDClassifier(random_state=1, max_iter=5, tol=None)
 sgd.fit(X_train, Y_train)
 acc_train_sgd = np.sqrt(mean_squared_log_error(Y_train, sgd.predict(X_train)))
 acc_eval_sgd = np.sqrt(mean_squared_log_error(Y_eval, sgd.predict(X_eval)))
 print(f'SGD: Train = {acc_train_sgd} Evaluate = {acc_eval_sgd}' )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
cv_score = 0
roc_score = 0
for i in range(len(target_cols)) :
cv_score += log_loss(y_true[:, i], y_pred[:, i])
 roc_score +=roc_auc_score(y_true[:, i], y_pred[:, i], average='micro')
 print("CV log_loss: ", cv_score / y_pred.shape[1])
 print("Overall AUC: ", roc_score / y_pred.shape[1] )<save_to_csv>
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 acc_train_rf = np.sqrt(mean_squared_log_error(Y_train, random_forest.predict(X_train)))
 acc_eval_rf = np.sqrt(mean_squared_log_error(Y_eval, random_forest.predict(X_eval)))
 print(f'RF: Train = {acc_train_rf} Evaluate = {acc_eval_rf}' )
oof_pretrain= valid_results[target_cols]
oof_pretrain.to_csv('off_pretrain_last.csv', index=False )<save_to_csv>
poly_features = PolynomialFeatures(degree=1)
 X_train_poly = poly_features.fit_transform(X_train)
 X_eval_poly = poly_features.fit_transform(X_eval)
 poly_model = LinearRegression()
poly_model.fit(X_train_poly, Y_train)
 acc_train_pr = np.sqrt(mean_squared_log_error(Y_train, poly_model.predict(X_train_poly)))
 acc_eval_pr = np.sqrt(mean_squared_log_error(Y_eval, abs(poly_model.predict(X_eval_poly))))
print(f'PR: Train = {acc_train_pr} Evaluate = {acc_eval_pr}' )
public_id = list(df['sig_id'].values)
 test_id = list(test_features['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=target_cols)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_submit.loc[test.sig_id,:] = test[target_cols].values
df_submit.loc[test_features[test_features.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )<install_modules>
logreg = LogisticRegression(random_state=1)
 logreg.fit(X_train, Y_train)
 acc_train_lr = np.sqrt(mean_squared_log_error(Y_train, logreg.predict(X_train)))
 acc_eval_lr = np.sqrt(mean_squared_log_error(Y_eval, logreg.predict(X_eval)))
 print(f'LR: Train = {acc_train_lr} Evaluate = {acc_eval_lr}' )
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null 
<set_options>
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 acc_train_gnb = np.sqrt(mean_squared_log_error(Y_train, gaussian.predict(X_train)))
 acc_eval_gnb = np.sqrt(mean_squared_log_error(Y_eval, gaussian.predict(X_eval)))
 print(f'GNB: Train = {acc_train_gnb} Evaluate = {acc_eval_gnb}' )
py.init_notebook_mode(connected=True)
 pd.set_option('max_columns', 50 )<define_variables>
perceptron = Perceptron(random_state=1, max_iter=50)
 perceptron.fit(X_train, Y_train)
 acc_train_prc = np.sqrt(mean_squared_log_error(Y_train, perceptron.predict(X_train)))
 acc_eval_prc = np.sqrt(mean_squared_log_error(Y_eval, perceptron.predict(X_eval)))
 print(f'Perceptron: Train = {acc_train_prc} Evaluate = {acc_eval_prc}' )
debug=False
submission=True
batch_size=256
device='cuda:0'
out='.'<define_variables>
linear_svc = LinearSVC(random_state=1)
 linear_svc.fit(X_train, Y_train)
 acc_train_svm = np.sqrt(mean_squared_log_error(Y_train, linear_svc.predict(X_train)))
 acc_eval_svm = np.sqrt(mean_squared_log_error(Y_eval, linear_svc.predict(X_eval)))
 print(f'SVM: Train = {acc_train_svm} Evaluate = {acc_eval_svm}' )
datadir = Path('/kaggle/input/bengaliai-cv19')
 featherdir = Path('/kaggle/input/bengaliaicv19feather')
 outdir = Path('.' )<load_from_csv>
decision_tree = DecisionTreeClassifier(random_state=1)
 decision_tree.fit(X_train, Y_train)
 acc_train_dt = np.sqrt(mean_squared_log_error(Y_train, decision_tree.predict(X_train)))
 acc_eval_dt = np.sqrt(mean_squared_log_error(Y_eval, decision_tree.predict(X_eval)))
 print(f'DT: Train = {acc_train_dt} Evaluate = {acc_eval_dt}' )
<categorify>
random_forest_reg = RandomForestRegressor(random_state=1, n_estimators=100)
 random_forest_reg.fit(X_train, Y_train)
 acc_train_rfr = np.sqrt(mean_squared_log_error(Y_train, random_forest_reg.predict(X_train)))
 acc_eval_rfr = np.sqrt(mean_squared_log_error(Y_eval, random_forest_reg.predict(X_eval)))
 print(f'RFR: Train = {acc_train_rfr} Evaluate = {acc_eval_rfr}' )
class DatasetMixin(Dataset):
def __init__(self, transform=None):
self.transform = transform
def __getitem__(self, index):

 if torch.is_tensor(index):
index = index.tolist()
if isinstance(index, slice):
current, stop, step = index.indices(len(self))
return [self.get_example_wrapper(i)for i in
six.moves.range(current, stop, step)]
elif isinstance(index, list)or isinstance(index, numpy.ndarray):
return [self.get_example_wrapper(i)for i in index]
else:
return self.get_example_wrapper(index)
 def __len__(self):

 raise NotImplementedError
def get_example_wrapper(self, i):

 example = self.get_example(i)
 if self.transform:
example = self.transform(example)
 return example
def get_example(self, i):

 raise NotImplementedError 
<categorify>
grad_boost = GradientBoostingRegressor(random_state=1, n_estimators=500, max_features= 0.3)
 grad_boost.fit(X_train, Y_train)
 acc_train_gb = np.sqrt(mean_squared_log_error(Y_train, grad_boost.predict(X_train)))
 acc_eval_gb = np.sqrt(mean_squared_log_error(Y_eval, grad_boost.predict(X_eval)))
 print(f'GB: Train = {acc_train_gb} Evaluate = {acc_eval_gb}' )
class BengaliAIDataset(DatasetMixin):
def __init__(self, images, labels=None, transform=None, indices=None):
super(BengaliAIDataset, self ).__init__(transform=transform)
 self.images = images
self.labels = labels
if indices is None:
indices = np.arange(len(images))
self.indices = indices
self.train = labels is not None
def __len__(self):

 return len(self.indices)
 def get_example(self, i):

 i = self.indices[i]
x = self.images[i]
x =(255 - x ).astype(np.float32)/ 255.
 if self.train:
y = self.labels[i]
return x, y
else:
return x 
<normalization>
ada_boost = AdaBoostRegressor(random_state=1, n_estimators=100)
 ada_boost.fit(X_train, Y_train)
 acc_train_ab = np.sqrt(mean_squared_log_error(Y_train, ada_boost.predict(X_train)))
 acc_eval_ab = np.sqrt(mean_squared_log_error(Y_eval, ada_boost.predict(X_eval)))
 print(f'AdaBoost: Train = {acc_train_ab} Evaluate = {acc_eval_ab}' )
def add_gaussian_noise(x, sigma):
x += np.random.randn(*x.shape)* sigma
x = np.clip(x, 0., 1.)
return x
class Transform:
def __init__(self, affine=True, crop=True, size=(64, 64),
normalize=True, train=True, threshold=40.,
sigma=-1.) :
self.affine = affine
self.crop = crop
self.size = size
self.normalize = normalize
self.train = train
self.threshold = threshold / 255.
 self.sigma = sigma / 255.
 def __call__(self, example):
if self.train:
x, y = example
else:
x = example
if self.affine:
x = affine_image(x)
 if self.crop:
x = crop_char_image(x, threshold=self.threshold)
 if self.size is not None:
x = resize(x, size=self.size)
 if self.sigma > 0.:
x = add_gaussian_noise(x, sigma=self.sigma)
 if self.normalize:
x =(x.astype(np.float32)- 0.0692)/ 0.2051
if x.ndim == 2:
x = x[None, :, :]
x = x.astype(np.float32)
 if self.train:
y = y.astype(np.int64)
 return x, y
else:
return x<concatenate>
ex_trees = ExtraTreesRegressor(random_state=1, n_estimators=100)
 ex_trees.fit(X_train, Y_train)
 acc_train_et = np.sqrt(mean_squared_log_error(Y_train, ex_trees.predict(X_train)))
 acc_eval_et = np.sqrt(mean_squared_log_error(Y_eval, ex_trees.predict(X_eval)))
 print(f'ET: Train = {acc_train_et} Evaluate = {acc_eval_et}' )
def residual_add(lhs, rhs):
lhs_ch, rhs_ch = lhs.shape[1], rhs.shape[1]
if lhs_ch < rhs_ch:
out = lhs + rhs[:, :lhs_ch]
elif lhs_ch > rhs_ch:
out = torch.cat([lhs[:, :rhs_ch] + rhs, lhs[:, rhs_ch:]], dim=1)
 else:
out = lhs + rhs
return out 
<import_modules>
reg1 = GradientBoostingRegressor(random_state=1, n_estimators=100)
 reg2 = RandomForestRegressor(random_state=1, n_estimators=100)
 reg3 = ExtraTreesRegressor(random_state=1, n_estimators=100)
 reg4 = AdaBoostRegressor(random_state=1, n_estimators=100)
 reg5 = xgb.XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=4, min_child_weight=1.5, n_estimators=2400,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 ereg = VotingRegressor(estimators=[('gb', reg1),('rf', reg2),('et', reg3),('ad', reg4),('xg', reg5)])
 ereg = ereg.fit(X_train, Y_train)
 acc_train_vr = np.sqrt(mean_squared_log_error(Y_train, ereg.predict(X_train)))
 acc_eval_vr = np.sqrt(mean_squared_log_error(Y_eval, ereg.predict(X_eval)))
 print(f'VR: Train = {acc_train_vr} Evaluate = {acc_eval_vr}' )
class LazyLoadModule(nn.Module):

 _lazy_buffer_keys: List[str] = []
_lazy_parameter_keys: List[str] = []
def __init__(self):
super(LazyLoadModule, self ).__init__()
for k in self._lazy_buffer_keys:
self.register_buffer(k, torch.tensor([]))
for k in self._lazy_parameter_keys:
self.register_parameter(k, None)
 self._register_load_state_dict_pre_hook(self._hook)
 def _hook(self, state_dict, prefix, local_metadata, strict, missing_keys,
unexpected_keys, error_msgs):
for key in self._lazy_buffer_keys:
self.register_buffer(key, state_dict[prefix + key])
 for key in self._lazy_parameter_keys:
self.register_parameter(key, Parameter(state_dict[prefix + key])) 
<import_modules>
est_cnt = 100
seed = 42
final_estimator = xgb.XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=4, min_child_weight=1.5, n_estimators=2400,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 final_layer = StackingRegressor(
 estimators=[ 
('abar', AdaBoostRegressor(random_state=seed, n_estimators=est_cnt))
,('adar', AdaBoostRegressor(random_state=seed, n_estimators=est_cnt))
,('gbrt', GradientBoostingRegressor(random_state=seed, n_estimators=est_cnt))
],
final_estimator=final_estimator 
)
 second_layer = StackingRegressor(
 estimators=[ 
('abar', AdaBoostRegressor(random_state=seed, n_estimators=est_cnt))
,('extr', ExtraTreesRegressor(random_state=seed, n_estimators=est_cnt))
,('adar', AdaBoostRegressor(random_state=seed, n_estimators=est_cnt))
,('gbrt', GradientBoostingRegressor(random_state=seed, n_estimators=est_cnt))
,('rf', RandomForestRegressor(random_state=seed, n_estimators=est_cnt))
],
final_estimator=final_layer 
)
 multi_layer_regressor = StackingRegressor(
 estimators=[ 
('lasso', LassoCV(random_state=seed))
,('svr', SVR(C=1, gamma=1e-6, kernel='rbf'))
,('etr', ExtraTreesRegressor(random_state=seed, n_estimators=est_cnt))
],
final_estimator=second_layer 
)
 multi_layer_regressor.fit(X_train, Y_train)
 acc_train_sg = np.sqrt(mean_squared_log_error(Y_train, multi_layer_regressor.predict(X_train)))
 acc_eval_sg = np.sqrt(mean_squared_log_error(Y_eval, multi_layer_regressor.predict(X_eval)))
 print(f'SG: Train = {acc_train_sg} Evaluate = {acc_eval_sg}' )
class LazyLinear(LazyLoadModule):

 __constants__ = ['bias', 'in_features', 'out_features']
_lazy_parameter_keys = ['weight']
def __init__(self, in_features, out_features, bias=True):
super(LazyLinear, self ).__init__()
self.in_features = in_features
self.out_features = out_features
if bias:
self.bias = Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
 if in_features is not None:
self.weight = Parameter(torch.Tensor(out_features, in_features))
self.reset_parameters()
def reset_parameters(self):
init.kaiming_uniform_(self.weight, a=math.sqrt(5))
if self.bias is not None:
fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight)
 bound = 1 / math.sqrt(fan_in)
 init.uniform_(self.bias, -bound, bound)
 def forward(self, input):
if self.weight is None:
self.in_features = input.shape[-1]
self.weight = Parameter(torch.Tensor(self.out_features, self.in_features))
self.reset_parameters()
self.to(input.device)
 return F.linear(input, self.weight, self.bias)
 def extra_repr(self):
return 'in_features={}, out_features={}, bias={}'.format(
 self.in_features, self.out_features, self.bias is not None 
)
<choose_model_class>
gbm = xgb.XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.7, gamma=2,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=5, min_child_weight=0, n_estimators=1900,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.3, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1 ).fit(X_train, Y_train)
 acc_train_xg = np.sqrt(mean_squared_log_error(Y_train, gbm.predict(X_train)))
 acc_eval_xg = np.sqrt(mean_squared_log_error(Y_eval, gbm.predict(X_eval)))
 print(f'XG: Train = {acc_train_xg} Evaluate = {acc_eval_xg}' )
<import_modules><EOS> 
best_reg = xgb.XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=5, min_child_weight=1.5, n_estimators=1900,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 best_reg.fit(X_train_full, Y_train_full)
 acc_train_full = np.sqrt(mean_squared_log_error(Y_train_full, best_reg.predict(X_train_full)))
 print(f'Best(previous): Train = {acc_train_xg} Evaluate = {acc_eval_xg}')
 print(f'Best FULL Train = {acc_train_full}')
 predictions = best_reg.predict(X_test)
 output = pd.DataFrame({'Id': test_df.Id, 'SalePrice': predictions})
 output.to_csv('my_submission.csv', index=False )