Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<compute_test_metric>
%matplotlib inline
output_results(gs_nn, x_test, df_test.listing_id.values, 'basic_nn.csv' )<load_from_disk>
train_data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test_data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv" )
random.seed(4321)
 np.random.seed(4321)
 X_train = pd.read_json(".. /input/train.json")
 X_test = pd.read_json(".. /input/test.json")
 interest_level_map = {'low': 0, 'medium': 1, 'high': 2}
X_train['interest_level'] = X_train['interest_level'].apply(lambda x: interest_level_map[x])
 X_test['interest_level'] = -1<feature_engineering>
train_data = train_data.drop(train_data[(train_data['OverallQual'] > 9)&(train_data['SalePrice'] < 220000)].index)
 train_data = train_data.drop(train_data[(train_data['GrLivArea'] > 4000)&(train_data['SalePrice'] < 300000)].index)
 train_data = train_data.drop(train_data[(train_data['GarageArea'] > 1200)&(train_data['SalePrice'] < 300000)].index )
feature_transform = CountVectorizer(stop_words='english', max_features=150)
 X_train['features'] = X_train["features"].apply(lambda x: " ".join(["_".join(i.lower().split(" ")) for i in x]))
X_test['features'] = X_test["features"].apply(lambda x: " ".join(["_".join(i.lower().split(" ")) for i in x]))
feature_transform.fit(list(X_train['features'])+ list(X_test['features']))
train_size = len(X_train)
 low_count = len(X_train[X_train['interest_level'] == 0])
 medium_count = len(X_train[X_train['interest_level'] == 1])
 high_count = len(X_train[X_train['interest_level'] == 2])
 def find_objects_with_only_one_record(feature_name):
temp = pd.concat([X_train[feature_name].reset_index() ,
X_test[feature_name].reset_index() ])
 temp = temp.groupby(feature_name, as_index = False ).count()
return temp[temp['index'] == 1]
managers_with_one_lot = find_objects_with_only_one_record('manager_id')
 buildings_with_one_lot = find_objects_with_only_one_record('building_id')
 addresses_with_one_lot = find_objects_with_only_one_record('display_address')
 lambda_val = None
k=5.0
f=1.0
r_k=0.01
g = 1.0
def categorical_average(variable, y, pred_0, feature_name):
def calculate_average(sub1, sub2):
s = pd.DataFrame(data = {
variable: sub1.groupby(variable, as_index = False ).count() [variable],
'sumy': sub1.groupby(variable, as_index = False ).sum() ['y'],
'avgY': sub1.groupby(variable, as_index = False ).mean() ['y'],
'cnt': sub1.groupby(variable, as_index = False ).count() ['y']
})
 tmp = sub2.merge(s.reset_index() , how='left', left_on=variable, right_on=variable)
 del tmp['index']
tmp.loc[pd.isnull(tmp['cnt']), 'cnt'] = 0.0
tmp.loc[pd.isnull(tmp['cnt']), 'sumy'] = 0.0
def compute_beta(row):
cnt = row['cnt'] if row['cnt'] < 200 else float('inf')
 return 1.0 /(g + exp(( cnt - k)/ f))
if lambda_val is not None:
tmp['beta'] = lambda_val
else:
tmp['beta'] = tmp.apply(compute_beta, axis = 1)
 tmp['adj_avg'] = tmp.apply(lambda row:(1.0 - row['beta'])* row['avgY'] + row['beta'] * row['pred_0'],
axis = 1)
 tmp.loc[pd.isnull(tmp['avgY']), 'avgY'] = tmp.loc[pd.isnull(tmp['avgY']), 'pred_0']
tmp.loc[pd.isnull(tmp['adj_avg']), 'adj_avg'] = tmp.loc[pd.isnull(tmp['adj_avg']), 'pred_0']
tmp['random'] = np.random.uniform(size = len(tmp))
tmp['adj_avg'] = tmp.apply(lambda row: row['adj_avg'] *(1 +(row['random'] - 0.5)* r_k),
axis = 1)
 return tmp['adj_avg'].ravel()
k_fold = StratifiedKFold(5)
 X_train[feature_name] = -999
for(train_index, cv_index)in k_fold.split(np.zeros(len(X_train)) ,
X_train['interest_level'].ravel()):
sub = pd.DataFrame(data = {variable: X_train[variable],
'y': X_train[y],
'pred_0': X_train[pred_0]})
 sub1 = sub.iloc[train_index]
sub2 = sub.iloc[cv_index]
X_train.loc[cv_index, feature_name] = calculate_average(sub1, sub2)
 sub1 = pd.DataFrame(data = {variable: X_train[variable],
'y': X_train[y],
'pred_0': X_train[pred_0]})
 sub2 = pd.DataFrame(data = {variable: X_test[variable],
'y': X_test[y],
'pred_0': X_test[pred_0]})
 X_test.loc[:, feature_name] = calculate_average(sub1, sub2)
 def transform_data(X):
feat_sparse = feature_transform.transform(X["features"])
 vocabulary = feature_transform.vocabulary_
del X['features']
X1 = pd.DataFrame([ pd.Series(feat_sparse[i].toarray().ravel())for i in np.arange(feat_sparse.shape[0])])
 X1.columns = list(sorted(vocabulary.keys()))
X = pd.concat([X.reset_index() , X1.reset_index() ], axis = 1)
 del X['index']
X["num_photos"] = X["photos"].apply(len)
 X['created'] = pd.to_datetime(X["created"])
 X["num_description_words"] = X["description"].apply(lambda x: len(x.split(" ")))
 X['price_per_bed'] = X['price'] / X['bedrooms']
X['price_per_bath'] = X['price'] / X['bathrooms']
X['price_per_room'] = X['price'] /(X['bathrooms'] + X['bedrooms'])
 X['low'] = 0
X.loc[X['interest_level'] == 0, 'low'] = 1
X['medium'] = 0
X.loc[X['interest_level'] == 1, 'medium'] = 1
X['high'] = 0
X.loc[X['interest_level'] == 2, 'high'] = 1
X['display_address'] = X['display_address'].apply(lambda x: x.lower().strip())
 X['street_address'] = X['street_address'].apply(lambda x: x.lower().strip())
 X['pred0_low'] = low_count * 1.0 / train_size
X['pred0_medium'] = medium_count * 1.0 / train_size
X['pred0_high'] = high_count * 1.0 / train_size
X.loc[X['manager_id'].isin(managers_with_one_lot['manager_id'].ravel()),
'manager_id'] = "-1"
X.loc[X['building_id'].isin(buildings_with_one_lot['building_id'].ravel()),
'building_id'] = "-1"
X.loc[X['display_address'].isin(addresses_with_one_lot['display_address'].ravel()),
'display_address'] = "-1"
return X
def normalize_high_cordiality_data() :
high_cardinality = ["building_id", "manager_id"]
for c in high_cardinality:
categorical_average(c, "medium", "pred0_medium", c + "_mean_medium")
 categorical_average(c, "high", "pred0_high", c + "_mean_high")
 def transform_categorical_data() :
categorical = ['building_id', 'manager_id',
'display_address', 'street_address']
for f in categorical:
encoder = LabelEncoder()
encoder.fit(list(X_train[f])+ list(X_test[f]))
X_train[f] = encoder.transform(X_train[f].ravel())
 X_test[f] = encoder.transform(X_test[f].ravel())
 def remove_columns(X):
columns = ["photos", "pred0_high", "pred0_low", "pred0_medium",
"description", "low", "medium", "high",
"interest_level", "created"]
for c in columns:
del X[c]
print("Starting transformations")
 X_train = transform_data(X_train)
 X_test = transform_data(X_test)
 y = X_train['interest_level'].ravel()
print("Normalizing high cordiality data...")
 normalize_high_cordiality_data()
transform_categorical_data()
remove_columns(X_train)
 remove_columns(X_test)
 print("Start fitting...")
 param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.02
param['max_depth'] = 4
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = 321
param['nthread'] = 8
num_rounds = 2000
xgtrain = xgb.DMatrix(X_train, label=y)
 clf = xgb.train(param, xgtrain, num_rounds)
 print("Fitted")
 def prepare_submission(model):
xgtest = xgb.DMatrix(X_test)
 preds = model.predict(xgtest)
 sub = pd.DataFrame(data = {'listing_id': X_test['listing_id'].ravel() })
 sub['low'] = preds[:, 0]
sub['medium'] = preds[:, 1]
sub['high'] = preds[:, 2]
sub.to_csv("submission.csv", index = False, header = True)
 prepare_submission(clf )<install_modules>
Target = 'SalePrice'
train_data.dropna(axis=0, subset=[Target], inplace=True)
 all_data = pd.concat([train_data, test_data],keys=['train','test'])
 print('У тренировочного датасета {} рядов и {} признаков'.format(train_data.shape[0], train_data.shape[1]))
print('У тестового датасета {} рядов и {} признаков'.format(test_data.shape[0], test_data.shape[1]))
print('Объединённый датасет содержит в себе {} рядов и {} признаков'.format(all_data.shape[0], all_data.shape[1]))
!pip install librosa<import_modules>
all_data = all_data.drop(columns=['Id'], axis=1 )
import numpy as np
import pandas as pd
import os<load_pretrained>
def missingValuesInfo(df):
total = df.isnull().sum().sort_values(ascending = False)
 percent = round(df.isnull().sum().sort_values(ascending = False)/ len(df)*100, 2)
 temp = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 return temp.loc[(temp['Total'] > 0)]
missingValuesInfo(train_data )
audio_data = '/kaggle/input/birdsong-recognition/train_audio/nutwoo/XC462016.mp3'
x , sr = librosa.load(audio_data)
 print(type(x), type(sr))
print(x.shape, sr )<load_pretrained>
def HandleMissingValues(df):
num_cols = [cname for cname in df.columns if df[cname].dtype in ['int64', 'float64']]
cat_cols = [cname for cname in df.columns if df[cname].dtype == "object"]
values = {}
for a in cat_cols:
values[a] = 'UNKNOWN'
for a in num_cols:
mean1=df[a].mean()
std1=df[a].std()
values[a]=random.randint(int(mean1-std1), int(mean1+std1))
df.fillna(value=values, inplace=True)
 HandleMissingValues(all_data)
 all_data.isnull().sum().sum()
librosa.load(audio_data, sr=44100 )<normalization>
def getObjectColumnsList(df):
return [cname for cname in df.columns if df[cname].dtype == "object"]
def PerformOneHotEncoding(df, columnsToEncode):
return pd.get_dummies(df, columns=columnsToEncode)
 cat_cols = getObjectColumnsList(all_data)
 all_data = PerformOneHotEncoding(all_data, cat_cols)
 all_data.head()
ipd.Audio(audio_data )<set_options>
target = train_data['SalePrice']
train_data = train_data.drop(['SalePrice'], axis=1)
 test_data = test_data.drop(['SalePrice'], axis=1)
 X, y = train_data, target
train_data.shape,test_data.shape
%matplotlib inline
plt.figure(figsize=(14, 5))
librosa.display.waveplot(x, sr=sr )<set_options>
gbr_reg = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,max_depth=4, max_features='sqrt',min_samples_leaf=15, min_samples_split=10,loss='huber')
 gbr_reg.fit(X, y)
 pred = gbr_reg.predict(X)
 print(gbr_reg.score(X, y))
np.sqrt(mean_squared_log_error(pred, y))
<import_modules><EOS> 
ourPrediction = gbr_reg.predict(test_data)
 submission = pd.DataFrame({
"Id": list(range(1461, 1461+len(test_data))),
"SalePrice": ourPrediction
})
 submission.to_csv('./submission.csv', index=False )