Kaggle competition: Tweet Sentiment Extraction
test_normality = lambda x: stats.shapiro(x.fillna(0)) [1] < 0.01
normal = pd.DataFrame(train[quantitative])
 normal = normal.apply(test_normality)
 print(not normal.any() )<categorify>
%matplotlib inline
def encode(frame, feature):
ordering = pd.DataFrame()
ordering['val'] = frame[feature].unique()
ordering.index = ordering.val
ordering['spmean'] = frame[[feature, 'SalePrice']].groupby(feature ).mean() ['SalePrice']
ordering = ordering.sort_values('spmean')
 ordering['ordering'] = range(1, ordering.shape[0]+1)
 ordering = ordering['ordering'].to_dict()
for cat, o in ordering.items() :
frame.loc[frame[feature] == cat, feature+'_E'] = o
qual_encoded = []
for q in qualitative:
encode(train, q)
 qual_encoded.append(q+'_E')
 print(qual_encoded )<normalization>
train = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/train.csv")
 test = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/test.csv")
 sample_submission = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/sample_submission.csv" )
features = quantitative + qual_encoded
model = TSNE(n_components=2, random_state=0, perplexity=50)
 X = train[features].fillna(0.).values
tsne = model.fit_transform(X)
 std = StandardScaler()
s = std.fit_transform(X)
 pca = PCA(n_components=30)
 pca.fit(s)
 pc = pca.transform(s)
 kmeans = KMeans(n_clusters=5)
 kmeans.fit(pc)
 fr = pd.DataFrame({'tsne1': tsne[:,0], 'tsne2': tsne[:, 1], 'cluster': kmeans.labels_})
 sns.lmplot(data=fr, x='tsne1', y='tsne2', hue='cluster', fit_reg=False)
 print(np.sum(pca.explained_variance_ratio_))<drop_column>
print(train[train["text"].isnull() == True])
 train.dropna(inplace = True )
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True )<prepare_x_and_y>
def len_sent(data):
return len(data.split())
 train["num_words_text"] = train["text"].apply(lambda x : len_sent(x))
test["num_words_text"] = test["text"].apply(lambda x : len_sent(x))
train["num_words_selected_text"] = train["selected_text"].apply(lambda x : len_sent(x))
train = train[train.GrLivArea < 4500]
train.reset_index(drop=True, inplace=True)
 train["SalePrice"] = np.log1p(train["SalePrice"])
 y = train['SalePrice'].reset_index(drop=True )<concatenate>
train["text"] = train["text"].apply(lambda x : x.strip())
 train["selected_text"] = train["selected_text"].apply(lambda x : x.strip())
 train["is_equal"] =(train["text"] == train["selected_text"])
 df_neutral = train[train["sentiment"] == "neutral"]
percentage =(len(df_neutral[df_neutral["is_equal"] == True])/len(df_neutral)) * 100
print(f"Percentage of text column sentences is equal selected_text column for neutral sentiment --> {percentage}" )
train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
features = pd.concat([train_features, test_features] ).reset_index(drop=True )<data_type_conversions>
def punc_count(data):
return len([w for w in data if w in string.punctuation])
 train["text_punc_count"] = train["text"].apply(lambda x : punc_count(x))
train["selected_text_punc_count"] = train["selected_text"].apply(lambda x : punc_count(x))
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str)
 features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features["PoolQC"] = features["PoolQC"].fillna("None")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
<categorify>
word_cloud(train[train["sentiment"] == "neutral"]["text"], "Most Repeated words in neural sentences train['text']" )
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))<data_type_conversions>
word_cloud(train[train["sentiment"] == "negative"]["text"], "Most Repeated words in negative sentences train['text']" )
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics.append(i)
 features.update(features[numerics].fillna(0))<feature_engineering>
word_cloud(train[train["sentiment"] == "positive"]["text"], "Most Repeated words in positive sentences train['text']" )
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics2.append(i)
 skew_features = features[numerics2].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))<feature_engineering>
def n_grams(ngram, data):
freq_dict = defaultdict(int)
 for text in data:
tokens = [w for w in text.lower().split() if w != " " if w not in stopwords]
ngrams = zip(*[tokens[i:] for i in range(ngram)])
 list_grams = [" ".join(ngram)for ngram in ngrams]
for word in list_grams:
freq_dict[word] += 1
fd_sorted = pd.DataFrame(sorted(freq_dict.items() , key=lambda x: x[1])[::-1])
 fd_sorted.columns = ["word", "wordcount"]
return fd_sorted
fd_sorted_neutral1 = n_grams(1, train[train["sentiment"] == "neutral"]["text"])
 fd_sorted_negative1 = n_grams(1, train[train["sentiment"] == "negative"]["text"])
 fd_sorted_positive1 = n_grams(1, train[train["sentiment"] == "positive"]["text"])
 fd_sorted_neutral2 = n_grams(2, train[train["sentiment"] == "neutral"]["text"])
 fd_sorted_negative2 = n_grams(2, train[train["sentiment"] == "negative"]["text"])
 fd_sorted_positive2 = n_grams(2, train[train["sentiment"] == "positive"]["text"])
 fd_sorted_neutral3 = n_grams(3, train[train["sentiment"] == "neutral"]["text"])
 fd_sorted_negative3 = n_grams(3, train[train["sentiment"] == "negative"]["text"])
 fd_sorted_positive3 = n_grams(3, train[train["sentiment"] == "positive"]["text"] )
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
<feature_engineering>
train_array = np.array(train.iloc[:, :4])
 test_array = np.array(test.iloc[:, :3])
 use_cuda = True
features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<categorify>
def start_index(text, selected_text):
start_index = text.lower().find(selected_text.lower())
 l.append(start_index)
 l = []
for i in range(len(train_array)) :
start_index(train_array[i, 1], train_array[i, 2] )
final_features = pd.get_dummies(features ).reset_index(drop=True)
 final_features.shape<train_model>
def quesa_format_train(train):
out = []
for i, row in enumerate(train):
qas = []
con = []
ans = []
question = row[-1]
answer = row[2]
context = row[1]
qid = row[0]
answer_start = l[i]
ans.append({"answer_start": answer_start, "text": answer.lower() })
 qas.append({"question": question, "id": qid, "is_impossible": False, "answers": ans})
 out.append({"context": context.lower() , "qas": qas})
 return out
train_json_format = quesa_format_train(train_array)
 with open('data/train.json', 'w')as outfile:
json.dump(train_json_format, outfile )
outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers])
 overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 X = X.drop(overfit, axis=1)
 X_sub = X_sub.drop(overfit, axis=1 )<compute_train_metric>
def quesa_format_test(train):
out = []
for i, row in enumerate(train):
qas = []
con = []
ans = []
question = row[-1]
context = row[1]
qid = row[0]
answer_start = l[i]
ans.append({"answer_start": 1000000, "text": "__None__"})
 qas.append({"question": question, "id": qid, "is_impossible": False, "answers": ans})
 out.append({"context": context.lower() , "qas": qas})
 return out
test_json_format = quesa_format_test(test_array)
 with open('data/test.json', 'w')as outfile:
json.dump(test_json_format, outfile )
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )<define_search_space>
!pip install '/kaggle/input/train-requirements/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/train-requirements/simpletransformers-0.22.1-py3-none-any.whl' -q
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]<define_search_model>
model_path = '/kaggle/input/transformers-pretrained-distilbert/distilbert-base-uncased-distilled-squad/'
model = QuestionAnsweringModel('distilbert',
model_path,
args={'reprocess_input_data': True,
'overwrite_output_dir': True,
'learning_rate': 5e-5,
'num_train_epochs': 4,
'max_seq_length': 128,
'doc_stride': 64,
'fp16': False,
},
use_cuda=use_cuda)
 model.train_model('data/train.json' )
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003,))<choose_model_class>
pred = model.predict(test_json_format )
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state =42 )<choose_model_class>
df = pd.DataFrame.from_dict(pred)
 sample_submission["selected_text"] = df["answer"]
sample_submission.to_csv("submission.csv", index = False)
 print("File submitted successfully." )
lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1,
)<choose_model_class>
%matplotlib inline
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006 )<choose_model_class>
train = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/train.csv")
 test = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/test.csv")
 sample_submission = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/sample_submission.csv" )
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True )<compute_test_metric>
print(train[train["text"].isnull() == True])
 train.dropna(inplace = True )
score = cv_rmse(ridge)
 score = cv_rmse(lasso)
 print("LASSO: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(elasticnet)
 print("elastic net: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(gbr)
 print("gbr: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(xgboost)
 print("xgboost: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )<train_model>
def len_sent(data):
return len(data.split())
 train["num_words_text"] = train["text"].apply(lambda x : len_sent(x))
test["num_words_text"] = test["text"].apply(lambda x : len_sent(x))
train["num_words_selected_text"] = train["selected_text"].apply(lambda x : len_sent(x))
print('START Fit')
 print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(y))
print('elasticnet')
 elastic_model_full_data = elasticnet.fit(X, y)
 print('Lasso')
 lasso_model_full_data = lasso.fit(X, y)
 print('Ridge')
 ridge_model_full_data = ridge.fit(X, y)
 print('Svr')
 svr_model_full_data = svr.fit(X, y)
 print('GradientBoosting')
 gbr_model_full_data = gbr.fit(X, y)
 print('xgboost')
 xgb_model_full_data = xgboost.fit(X, y)
 print('lightgbm')
 lgb_model_full_data = lightgbm.fit(X, y )<predict_on_test>
train["text"] = train["text"].apply(lambda x : x.strip())
 train["selected_text"] = train["selected_text"].apply(lambda x : x.strip())
 train["is_equal"] =(train["text"] == train["selected_text"])
 df_neutral = train[train["sentiment"] == "neutral"]
percentage =(len(df_neutral[df_neutral["is_equal"] == True])/len(df_neutral)) * 100
print(f"Percentage of text column sentences is equal selected_text column for neutral sentiment --> {percentage}" )
def blend_models_predict(X):
return(( 0.1 * elastic_model_full_data.predict(X)) + \ 
(0.05 * lasso_model_full_data.predict(X)) + \ 
(0.1 * ridge_model_full_data.predict(X)) + \ 
(0.1 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.15 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.3 * stack_gen_model.predict(np.array(X))))<compute_test_metric>
def punc_count(data):
return len([w for w in data if w in string.punctuation])
 train["text_punc_count"] = train["text"].apply(lambda x : punc_count(x))
train["selected_text_punc_count"] = train["selected_text"].apply(lambda x : punc_count(x))
print('RMSLE score on train data:')
 print(rmsle(y, blend_models_predict(X)) )<load_from_csv>
word_cloud(train[train["sentiment"] == "neutral"]["text"], "Most Repeated words in neural sentences train['text']" )
print('Predict submission')
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_sub)) )<load_from_csv>
word_cloud(train[train["sentiment"] == "negative"]["text"], "Most Repeated words in negative sentences train['text']" )
print('Blend with Top Kernels submissions
')
 sub_1 = pd.read_csv('.. /input/top-10-0-10943-stacking-mice-and-brutal-force/House_Prices_submit.csv')
 sub_2 = pd.read_csv('.. /input/hybrid-svm-benchmark-approach-0-11180-lb-top-2/hybrid_solution.csv')
 sub_3 = pd.read_csv('.. /input/lasso-model-for-regression-problem/lasso_sol22_Median.csv')
 submission.iloc[:,1] = np.floor(( 0.25 * np.floor(np.expm1(blend_models_predict(X_sub)))) + 
(0.25 * sub_1.iloc[:,1])+ 
(0.25 * sub_2.iloc[:,1])+ 
(0.25 * sub_3.iloc[:,1]))<save_to_csv>
word_cloud(train[train["sentiment"] == "positive"]["text"], "Most Repeated words in positive sentences train['text']" )
q1 = submission['SalePrice'].quantile(0.005)
 q2 = submission['SalePrice'].quantile(0.995)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.to_csv("submission.csv", index=False )<import_modules>
def n_grams(ngram, data):
freq_dict = defaultdict(int)
 for text in data:
tokens = [w for w in text.lower().split() if w != " " if w not in stopwords]
ngrams = zip(*[tokens[i:] for i in range(ngram)])
 list_grams = [" ".join(ngram)for ngram in ngrams]
for word in list_grams:
freq_dict[word] += 1
fd_sorted = pd.DataFrame(sorted(freq_dict.items() , key=lambda x: x[1])[::-1])
 fd_sorted.columns = ["word", "wordcount"]
return fd_sorted
fd_sorted_neutral1 = n_grams(1, train[train["sentiment"] == "neutral"]["text"])
 fd_sorted_negative1 = n_grams(1, train[train["sentiment"] == "negative"]["text"])
 fd_sorted_positive1 = n_grams(1, train[train["sentiment"] == "positive"]["text"])
 fd_sorted_neutral2 = n_grams(2, train[train["sentiment"] == "neutral"]["text"])
 fd_sorted_negative2 = n_grams(2, train[train["sentiment"] == "negative"]["text"])
 fd_sorted_positive2 = n_grams(2, train[train["sentiment"] == "positive"]["text"])
 fd_sorted_neutral3 = n_grams(3, train[train["sentiment"] == "neutral"]["text"])
 fd_sorted_negative3 = n_grams(3, train[train["sentiment"] == "negative"]["text"])
 fd_sorted_positive3 = n_grams(3, train[train["sentiment"] == "positive"]["text"] )
from datetime import datetime
from scipy.stats import skew
from scipy.special import boxcox1p
from scipy.stats import boxcox_normmax
from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error
from mlxtend.regressor import StackingCVRegressor 
<load_from_csv>
train_array = np.array(train.iloc[:, :4])
 test_array = np.array(test.iloc[:, :3])
 use_cuda = True
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 print("Train set size:", train.shape)
 print("Test set size:", test.shape)
 print('START data processing', datetime.now() ,)
 train_ID = train['Id']
test_ID = test['Id']
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True)
 train = train[train.GrLivArea < 4500]
train.reset_index(drop=True, inplace=True)
 train["SalePrice"] = np.log1p(train["SalePrice"])
 y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 print(features.shape)
 features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str)
 features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics.append(i)
 features.update(features[numerics].fillna(0))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics2.append(i)
 skew_features = features[numerics2].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 print(features.shape)
 final_features = pd.get_dummies(features ).reset_index(drop=True)
 print(final_features.shape)
 X = final_features.iloc[:len(y), :]
X_sub = final_features.iloc[len(X):, :]
print('X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape)
 outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers])
 overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 overfit.append('MSZoning_C(all)')
 X = X.drop(overfit, axis=1 ).copy()
X_sub = X_sub.drop(overfit, axis=1 ).copy()
print('X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape)
 print('START ML', datetime.now() ,)
 kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y,
scoring="neg_mean_squared_error",
cv=kfolds))
return(rmse)
 alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7, alphas=alphas2,
random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() ,
SVR(C= 20, epsilon= 0.008, gamma=0.0003,))
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =42)
 lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 xgboost = XGBRegressor(learning_rate=0.01, n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet,
gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True)
 print('TEST score on CV')
 score = cv_rmse(ridge)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lasso)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(elasticnet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(svr)
 print("SVR score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lightgbm)
 print("Lightgbm score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(gbr)
 print("GradientBoosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(xgboost)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 print('START Fit')
 print(datetime.now() , 'StackingCVRegressor')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(y))
print(datetime.now() , 'elasticnet')
 elastic_model_full_data = elasticnet.fit(X, y)
 print(datetime.now() , 'lasso')
 lasso_model_full_data = lasso.fit(X, y)
 print(datetime.now() , 'ridge')
 ridge_model_full_data = ridge.fit(X, y)
 print(datetime.now() , 'svr')
 svr_model_full_data = svr.fit(X, y)
 print(datetime.now() , 'GradientBoosting')
 gbr_model_full_data = gbr.fit(X, y)
 print(datetime.now() , 'xgboost')
 xgb_model_full_data = xgboost.fit(X, y)
 print(datetime.now() , 'lightgbm')
 lgb_model_full_data = lightgbm.fit(X, y)
<predict_on_test>
def start_index(text, selected_text):
start_index = text.lower().find(selected_text.lower())
 l.append(start_index)
 l = []
for i in range(len(train_array)) :
start_index(train_array[i, 1], train_array[i, 2] )
def blend_models_predict(X):
return(( 0.1 * elastic_model_full_data.predict(X)) + \ 
(0.1 * lasso_model_full_data.predict(X)) + \ 
(0.1 * ridge_model_full_data.predict(X)) + \ 
(0.1 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.15 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.25 * stack_gen_model.predict(np.array(X))))
print('RMSLE score on train data:')
 print(rmsle(y, blend_models_predict(X)) )<load_from_csv>
def quesa_format_train(train):
out = []
for i, row in enumerate(train):
qas = []
con = []
ans = []
question = row[-1]
answer = row[2]
context = row[1]
qid = row[0]
answer_start = l[i]
ans.append({"answer_start": answer_start, "text": answer.lower() })
 qas.append({"question": question, "id": qid, "is_impossible": False, "answers": ans})
 out.append({"context": context.lower() , "qas": qas})
 return out
train_json_format = quesa_format_train(train_array)
 with open('data/train.json', 'w')as outfile:
json.dump(train_json_format, outfile )
print('Predict submission', datetime.now() ,)
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_sub)))
 print('Blend with Top Kernals submissions', datetime.now() ,)
 sub_1 = pd.read_csv('.. /input/top-10-0-10943-stacking-mice-and-brutal-force/House_Prices_submit.csv')
 sub_2 = pd.read_csv('.. /input/hybrid-svm-benchmark-approach-0-11180-lb-top-2/hybrid_solution.csv')
 sub_3 = pd.read_csv('.. /input/lasso-model-for-regression-problem/lasso_sol22_Median.csv')
 submission.iloc[:,1] = np.floor(( 0.25 * np.floor(np.expm1(blend_models_predict(X_sub)))) + 
(0.25 * sub_1.iloc[:,1])+ 
(0.25 * sub_2.iloc[:,1])+ 
(0.25 * sub_3.iloc[:,1]))<save_to_csv>
def quesa_format_test(train):
out = []
for i, row in enumerate(train):
qas = []
con = []
ans = []
question = row[-1]
context = row[1]
qid = row[0]
answer_start = l[i]
ans.append({"answer_start": 1000000, "text": "__None__"})
 qas.append({"question": question, "id": qid, "is_impossible": False, "answers": ans})
 out.append({"context": context.lower() , "qas": qas})
 return out
test_json_format = quesa_format_test(test_array)
 with open('data/test.json', 'w')as outfile:
json.dump(test_json_format, outfile )
q1 = submission['SalePrice'].quantile(0.0045)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.to_csv("new_submission.csv", index=False)
 print('Save submission', datetime.now() , )<save_to_csv>
!pip install '/kaggle/input/train-requirements/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/train-requirements/simpletransformers-0.22.1-py3-none-any.whl' -q
submission = pd.read_csv('.. /input/my-best-house-price/House_0.10649.csv')
 submission.to_csv('best_submission.csv', index=False )<import_modules>
model_path = '/kaggle/input/transformers-pretrained-distilbert/distilbert-base-uncased-distilled-squad/'
model = QuestionAnsweringModel('distilbert',
model_path,
args={'reprocess_input_data': True,
'overwrite_output_dir': True,
'learning_rate': 5e-5,
'num_train_epochs': 4,
'max_seq_length': 128,
'doc_stride': 64,
'fp16': False,
},
use_cuda=use_cuda)
 model.train_model('data/train.json' )
import pandas as pd<import_modules>
pred = model.predict(test_json_format )
<load_from_csv><EOS> 
df = pd.DataFrame.from_dict(pred)
 sample_submission["selected_text"] = df["answer"]
sample_submission.to_csv("submission.csv", index = False)
 print("File submitted successfully." )