Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<choose_model_class>
pd.plotting.register_matplotlib_converters()
%matplotlib inline
model = start_model(model_name,model_shortcut_name)
 optimizer,scheduler = get_optimizer_and_scheduler(learning_rate,full_train_dataloader,best_epoch )<define_variables>
X_path = '.. /input/house-prices-advanced-regression-techniques/train.csv'
X = pd.read_csv(X_path, index_col='Id')
 test_path = ".. /input/house-prices-advanced-regression-techniques/test.csv"
X_test = pd.read_csv(test_path, index_col='Id')
 X.dropna(axis=0, subset=['SalePrice'], inplace=True)
 y = X.SalePrice
X.drop(['SalePrice'], axis=1, inplace=True)
 X.head()
seed_val = 66
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 training_stats = []
total_t0 = time.time()
for epoch_i in range(0, best_epoch):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, best_epoch))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
for step, batch in enumerate(full_train_dataloader):
if step % 40 == 0 :
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_time = format_time(time.time() - t0)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
training_stats.append(
 {
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Training Time': training_time,
} 
)
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)) )<load_from_csv>
num_cols = [col for col in X.columns if X[col].dtypes in ['int64', 'float64']]
print(num_cols)
 print()
print(len(num_cols))
df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 print('Number of test sentences: {:,}
'.format(df.shape[0]))
sentences = df.text.values
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 75,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 prediction_data = TensorDataset(input_ids, attention_masks)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size )<predict_on_test>
cat_cols = [col for col in X.columns if X[col].dtypes=='object']
print(cat_cols)
 print()
print(len(cat_cols))
print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))
 model.eval()
predictions , true_labels = [], []
for batch in prediction_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 print(' DONE.' )<prepare_output>
cols_with_missing = [col for col in X.columns if X[col].isnull().any() ]
print(cols_with_missing)
 print()
print(len(cols_with_missing))
cols_with_missing_test = [col for col in X_test.columns if X_test[col].isnull().any() ]
print(cols_with_missing_test)
 print()
print(len(cols_with_missing_test))
flat_predictions = np.concatenate(predictions, axis=0)
 flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
df['target'] = flat_predictions 
<save_to_csv>
for col in cols_with_missing:
print(col + " : " + str(X[col].isnull().sum()))
submission['target'] = flat_predictions
submission.to_csv('torch_prediction.csv', index=False )<save_to_csv>
X.drop(['Alley', 'PoolQC', 'Fence', 'MiscFeature'], axis=1, inplace=True)
 X_test.drop(['Alley', 'PoolQC', 'Fence', 'MiscFeature'], axis=1, inplace=True )
<train_model>
cols_with_missing = [col for col in X.columns if X[col].isnull().any() ]
cat_cols = [col for col in X.columns if X[col].dtypes=='object']
cols_with_missing_test = [col for col in X_test.columns if X_test[col].isnull().any() ]
cat_cols_test = [col for col in X_test.columns if X_test[col].dtypes=='object']
<define_variables>
X.drop(['LotConfig', 'LandSlope'], axis=1, inplace=True)
 X_test.drop(['LotConfig', 'LandSlope'], axis=1, inplace=True)
 cat_cols = [col for col in X.columns if X[col].dtypes=='object']
cat_cols_test = [col for col in X_test.columns if X_test[col].dtypes=='object']
<import_modules>
X.drop(['YrSold', 'MoSold', 'MiscVal', '3SsnPorch', 'BsmtHalfBath', 'LowQualFinSF', 'BsmtFinSF2', 'MSSubClass'],
axis=1, inplace=True)
 X_test.drop(['YrSold', 'MoSold', 'MiscVal', '3SsnPorch', 'BsmtHalfBath', 'LowQualFinSF', 'BsmtFinSF2', 'MSSubClass'],
axis=1, inplace=True)
 num_cols = [col for col in X.columns if X[col].dtypes in ['int64', 'float64']]
num_cols_test = [col for col in X_test.columns if X_test[col].dtypes in ['int64', 'float64']]
<load_from_csv>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=0 )
<import_modules>
pd.options.mode.chained_assignment = None
seed(1)
<load_from_csv>
cat_cols_with_missing = [col for col in cat_cols if col in cols_with_missing]
cat_cols_with_missing_test = [col for col in cat_cols_test if col in cols_with_missing_test]
cat_imputer = SimpleImputer(strategy='most_frequent')
 X_train_cat_imputed = pd.DataFrame(cat_imputer.fit_transform(X_train[cat_cols_with_missing]))
X_valid_cat_imputed = pd.DataFrame(cat_imputer.transform(X_valid[cat_cols_with_missing]))
X_train_cat_imputed.columns = X_train[cat_cols_with_missing].columns
X_valid_cat_imputed.columns = X_valid[cat_cols_with_missing].columns
X_train_cat_imputed.index = X_train.index
X_valid_cat_imputed.index = X_valid.index
X_train.drop(cat_cols_with_missing, axis=1, inplace=True)
 X_valid.drop(cat_cols_with_missing, axis=1, inplace=True)
 X_train_no_missing_cat = pd.concat([X_train, X_train_cat_imputed], axis=1)
 X_valid_no_missing_cat = pd.concat([X_valid, X_valid_cat_imputed], axis=1)
 cat_imputer_for_test = SimpleImputer(strategy='most_frequent')
 cat_imputer_for_test.fit(X_train_no_missing_cat[cat_cols_with_missing_test])
 X_test_cat_imputed = pd.DataFrame(cat_imputer_for_test.transform(X_test[cat_cols_with_missing_test]))
X_test_cat_imputed.columns = X_test[cat_cols_with_missing_test].columns
X_test_cat_imputed.index = X_test.index
X_test.drop(cat_cols_with_missing_test, axis=1, inplace=True)
 X_test_no_missing_cat = pd.concat([X_test, X_test_cat_imputed], axis=1 )
tweet = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 testset = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 tweet.head()<string_transform>
cb_enc = ce.CatBoostEncoder(cols=cat_cols)
 cb_enc.fit(X_train_no_missing_cat[cat_cols], y_train)
 train_encoded_cb = X_train_no_missing_cat.join(cb_enc.transform(X_train_no_missing_cat[cat_cols] ).add_suffix('_cb'))
valid_encoded_cb = X_valid_no_missing_cat.join(cb_enc.transform(X_valid_no_missing_cat[cat_cols] ).add_suffix('_cb'))
test_encoded_cb = X_test_no_missing_cat.join(cb_enc.transform(X_test_no_missing_cat[cat_cols] ).add_suffix('_cb'))
train_encoded_cb.drop(cat_cols, axis=1, inplace=True)
 valid_encoded_cb.drop(cat_cols, axis=1, inplace=True)
 test_encoded_cb.drop(cat_cols, axis=1, inplace=True )
def create_corpus() :
corpus=[]
for x in tweet['text'].str.split() :
for i in x:
corpus.append(i)
 return corpus<feature_engineering>
imputer = KNNImputer(n_neighbors=100, weights='distance', metric='nan_euclidean')
 imputed_train_num_cols = pd.DataFrame(imputer.fit_transform(train_encoded_cb[num_cols]))
imputed_valid_num_cols = pd.DataFrame(imputer.transform(valid_encoded_cb[num_cols]))
imputed_test_num_cols = pd.DataFrame(imputer.transform(test_encoded_cb[num_cols]))
imputed_train_num_cols.columns = train_encoded_cb[num_cols].columns
imputed_valid_num_cols.columns = valid_encoded_cb[num_cols].columns
imputed_test_num_cols.columns = test_encoded_cb[num_cols].columns
imputed_train_num_cols.index = train_encoded_cb[num_cols].index
imputed_valid_num_cols.index = valid_encoded_cb[num_cols].index
imputed_test_num_cols.index = test_encoded_cb[num_cols].index
train_encoded_cb.drop(num_cols, axis=1, inplace=True)
 valid_encoded_cb.drop(num_cols, axis=1, inplace=True)
 test_encoded_cb.drop(num_cols, axis=1, inplace=True)
 train_data = train_encoded_cb.join(imputed_train_num_cols)
 valid_data = valid_encoded_cb.join(imputed_valid_num_cols)
 test_data = test_encoded_cb.join(imputed_test_num_cols )
def get_top_tweet_bigrams(corpus, n=None):
vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq[:n]<feature_engineering>
ada_boost = AdaBoostRegressor(base_estimator=XGBRegressor(verbosity=0))
ada_boost.fit(train_data, y_train )
def build_vocab(X):
tweets = X.apply(lambda s: s.split() ).values
vocab = {}
for tweet in tweets:
for word in tweet:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def check_embeddings_coverage(X, embeddings):
vocab = build_vocab(X)
 covered = {}
oov = {}
n_covered = 0
n_oov = 0
for word in vocab:
try:
covered[word] = embeddings[word]
n_covered += vocab[word]
except:
oov[word] = vocab[word]
n_oov += vocab[word]
vocab_coverage = len(covered)/ len(vocab)
 text_coverage =(n_covered /(n_covered + n_oov))
sorted_oov = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_oov, vocab_coverage, text_coverage<load_from_csv>
xgb_model = XGBRegressor()
xgb_model.fit(train_data, y_train )
embedding_dict={}
with open('.. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt','r')as f:
for line in f:
values=line.split()
word=values[0]
vectors=np.asarray(values[1:],'float32')
 embedding_dict[word]=vectors
f.close()<compute_train_metric>
lgb_model = lgb.LGBMRegressor()
lgb_model.fit(train_data, y_train )
train_glove_oov, train_glove_vocab_coverage, train_glove_text_coverage = check_embeddings_coverage(tweet["text"], embedding_dict)
 test_glove_oov, test_glove_vocab_coverage, test_glove_text_coverage = check_embeddings_coverage(testset['text'], embedding_dict)
 print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Training Set'.format(train_glove_vocab_coverage, train_glove_text_coverage))
print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Test Set'.format(test_glove_vocab_coverage, test_glove_text_coverage)) 
<categorify>
rf_model = RandomForestRegressor()
rf_model.fit(train_data, y_train )
def utils_preprocess_text(text):
text = re.sub(r'[^\w\s]', '', str(text ).lower().strip())
 url = re.compile(r'https?://\S+|www\.\S+')
 text = url.sub(r'',text)
 url = re.compile(r'http?://\S+|www\.\S+')
 text = url.sub(r'',text)
 html=re.compile(r'<.*?>')
 html.sub(r'',text)
 text = re.sub(r'mh370','flight crash',text)
 text = re.sub(r'û_','',text)
 text = re.sub(r'ûò','',text)
 text = re.sub(r'typhoondevastated','typhoon devastated',text)
 text = re.sub(r'irandeal','iran deal',text)
 text = re.sub(r'worldnews','world news',text)
 text = re.sub(r'animalrescue','animal rescue',text)
 text = re.sub(r'viralspell','viral spell',text)
 text = re.sub(r'griefûª','grief',text)
 text = re.sub(r'pantherattack','panther attack',text)
 text = re.sub(r'injuryi495','injury in 495',text)
 text = re.sub(r'explosionproof','explosion proof',text)
 text = re.sub(r'americaûªs','americans',text)
 return text 
<concatenate>
gb_model = GradientBoostingRegressor()
gb_model.fit(train_data, y_train )
df=pd.concat([tweet,testset])
 df.shape<feature_engineering>
et_model = ExtraTreesRegressor()
et_model.fit(train_data, y_train )
df["text_clean"] = df["text"].apply(lambda x: utils_preprocess_text(x))
df.head()<compute_test_metric>
print("AdaBoost Score : ",ada_boost.score(valid_data, y_valid))
print("XGBoost Score : ",xgb_model.score(valid_data, y_valid))
print("LightGBM Score : ",lgb_model.score(valid_data, y_valid))
print("Random Forests Score : ",rf_model.score(valid_data, y_valid))
print("Gradient Boosting Score: ",gb_model.score(valid_data, y_valid))
print("Extra Trees Score : ",et_model.score(valid_data, y_valid))
df_glove_oov, df_glove_vocab_coverage, df_glove_text_coverage = check_embeddings_coverage(df["text_clean"], embedding_dict)
 print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Training Set'.format(df_glove_vocab_coverage, df_glove_text_coverage)) 
<define_variables>
params_for_ada_boost = {
"base_estimator__n_estimators": [100, 300, 500],
"base_estimator__reg_lambda": [1, 2, 4],
"n_estimators": [10, 30, 50],
"learning_rate": [0.05, 0.1],
}
corpus = []
corpus = df["text_clean"]<categorify>
random_search_ada_boost = RandomizedSearchCV(
 estimator=ada_boost,
param_distributions=params_for_ada_boost,
scoring='neg_mean_absolute_error',
n_iter=5,
n_jobs=-1,
cv=3,
verbose=True
)
MAX_LEN=50
tokenizer_obj=Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences=tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad=pad_sequences(sequences,maxlen=MAX_LEN,truncating='post',padding='post' )<count_unique_values>
random_search_ada_boost.fit(train_data, y_train )
word_index=tokenizer_obj.word_index
print('Number of unique words:',len(word_index))<define_variables>
-1*random_search_ada_boost.best_score_
num_words=len(word_index)+1
embedding_matrix=np.zeros(( num_words,100))
for word,i in tqdm(word_index.items()):
if i < num_words:
emb_vec=embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i]=emb_vec<choose_model_class>
random_search_ada_boost.best_estimator_
model=Sequential()
embedding=Embedding(num_words,100,embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN,trainable=False)
 model.add(embedding)
 model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
optimzer=Adam(learning_rate=3e-4)
 model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'] )<prepare_x_and_y>
ada_boost_tuned = AdaBoostRegressor(base_estimator=XGBRegressor(base_score=0.5, booster='gbtree',
colsample_bylevel=1,
colsample_bynode=1,
colsample_bytree=1, gamma=0,
importance_type='gain',
learning_rate=0.1,
max_delta_step=0, max_depth=3,
min_child_weight=1, missing=None,
n_estimators=500, n_jobs=1,
nthread=None,
objective='reg:linear',
random_state=0, reg_alpha=0,
reg_lambda=2, scale_pos_weight=1,
seed=None, silent=None,
subsample=1, verbosity=0),
learning_rate=0.1, loss='linear', n_estimators=10,
random_state=None)
 ada_boost_tuned.fit(train_data, y_train )
train=tweet_pad[:tweet.shape[0]]
test=tweet_pad[tweet.shape[0]:]<split>
ada_boost_tuned.score(valid_data, y_valid )
X_train,X_test,y_train,y_test=train_test_split(train,tweet['target'].values,test_size=0.2)
 print('Shape of train',X_train.shape)
 print("Shape of Validation ",X_test.shape )<train_model>
params_for_xgb_model = {
'n_estimators': [500, 1000],
'learning_rate': [0.03, 0.05, 0.1],
'max_depth': [3, 6, 9],
'gamma': [1,5]
}
history=model.fit(X_train,y_train,batch_size=32,epochs=10,validation_data=(X_test,y_test),verbose=2 )<predict_on_test>
random_search_xgb_model = RandomizedSearchCV(
 estimator=xgb_model,
param_distributions=params_for_xgb_model,
scoring='neg_mean_absolute_error',
n_iter=5,
n_jobs=-1,
cv=3,
verbose=True
)
train_pred_GloVe = model.predict(train)
 train_pred_GloVe_int = train_pred_GloVe.round().astype('int' )<save_to_csv>
random_search_xgb_model.fit(train_data, y_train )
test_pred_GloVe = model.predict(test)
 test_pred_GloVe_int = test_pred_GloVe.round().astype('int')
 submission['target'] = test_pred_GloVe_int
submission.head(10)
 submission.to_csv("submission.csv", index=False, header=True)
<install_modules>
-1*random_search_xgb_model.best_score_
!pip -q install pyspellchecker optuna<import_modules>
random_search_xgb_model.best_estimator_
import os
import optuna
import pandas as pd
import numpy as np
import random
import re
from scipy import sparse
from spellchecker import SpellChecker
import string
import warnings
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.utils.validation import check_X_y, check_is_fitted
from sklearn.model_selection import StratifiedKFold, train_test_split
from sklearn.metrics import accuracy_score, matthews_corrcoef, f1_score<set_options>
xgb_model_tuned = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=1,
importance_type='gain', learning_rate=0.03, max_delta_step=0,
max_depth=3, min_child_weight=1, missing=None, n_estimators=1000,
n_jobs=1, nthread=None, objective='reg:linear', random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=None, subsample=1, verbosity=1)
 xgb_model_tuned.fit(train_data, y_train )
optuna.logging.set_verbosity(0)
 warnings.filterwarnings("ignore" )<define_variables>
xgb_model_tuned.score(valid_data, y_valid )
SEED = 42
NUM_SPLITS = 10
NUM_TRIALS = 100<define_variables>
params_for_lgb_model = {
'n_estimators': [600, 800, 1000],
'learning_rate': [0.03, 0.05],
}
os.environ['PYTHONHASHSEED']=str(SEED)
 random.seed(SEED)
 np.random.seed(SEED )<load_from_csv>
random_search_lgb_model = RandomizedSearchCV(
 estimator=lgb_model,
param_distributions=params_for_lgb_model,
scoring='neg_mean_absolute_error',
n_iter=5,
n_jobs=-1,
cv=3,
verbose=True
)
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv" )<concatenate>
random_search_lgb_model.fit(train_data, y_train )
test["target"] = -1
df = pd.concat([train, test] )<count_missing_values>
-1*random_search_lgb_model.best_score_
print("NaN Distribution
")
 for col in df.columns:
print(f"{col}: {(( df[col].isna().sum() /df.shape[0])*100):.2f}" )<feature_engineering>
random_search_lgb_model.best_estimator_
df["text"] = df["text"].str.lower()<feature_engineering>
lgb_model_tuned = lgb.LGBMRegressor(boosting='gbdt', boosting_type='gbdt', class_weight=None,
colsample_bytree=1.0, importance_type='split', learning_rate=0.05,
max_depth=-1, min_child_samples=20, min_child_weight=0.001,
min_split_gain=0.0, n_estimators=1000, n_jobs=-1, num_leaves=31,
objective=None, random_state=None, reg_alpha=0.0, reg_lambda=0.0,
silent=True, subsample=1.0, subsample_for_bin=200000,
subsample_freq=0)
 lgb_model_tuned.fit(valid_data, y_valid )
PUNCT_TO_REMOVE = string.punctuation
def remove_punctuation(text):
return text.translate(str.maketrans('', '', PUNCT_TO_REMOVE))
df["text"] = df["text"].apply(lambda text: remove_punctuation(text))<drop_column>
lgb_model_tuned.score(valid_data, y_valid )
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 df["text"] = df["text"].apply(lambda text: remove_emoji(text))<feature_engineering>
params_for_rf_model = {
'n_estimators': [500, 1000, 1500, 2000]
}
def remove_urls(text):
url_pattern = re.compile(r'https?://\S+|www\.\S+')
 try:
return url_pattern.sub(r'', text)
 except:
print(text)
 df["text"] = df["text"].apply(lambda text: remove_urls(text))<feature_engineering>
random_search_rf_model = RandomizedSearchCV(
 estimator=rf_model,
param_distributions=params_for_rf_model,
scoring='neg_mean_absolute_error',
n_iter=5,
n_jobs=-1,
cv=3,
verbose=True
)
def remove_html(text):
html_pattern = re.compile('<.*?>')
 return html_pattern.sub(r'', text)
 df["text"] = df["text"].apply(lambda text: remove_html(text))<feature_engineering>
random_search_rf_model.fit(train_data, y_train )
with open(".. /input/slangtext/slang.txt", "r")as file:
chat_words_str = file.read()
chat_words_map_dict = {}
chat_words_list = []
for line in chat_words_str.split("
"):
if line != "" and "=" in line:
cw = line.split("=")[0]
cw_expanded = line.split("=")[1]
chat_words_list.append(cw)
 chat_words_map_dict[cw] = cw_expanded
chat_words_list = set(chat_words_list )<feature_engineering>
-1*random_search_rf_model.best_score_
def chat_words_conversion(text):
new_text = []
for w in text.split() :
if w.upper() in chat_words_list:
new_text.append(chat_words_map_dict[w.upper() ])
 else:
new_text.append(w)
 return " ".join(new_text)
 df["text"] = df["text"].apply(lambda text: chat_words_conversion(text))<feature_engineering>
random_search_rf_model.best_estimator_
spell = SpellChecker()
def correct_spellings(text):
corrected_text = []
misspelled_words = spell.unknown(text.split())
 for word in text.split() :
if word in misspelled_words:
corrected_text.append(spell.correction(word))
else:
corrected_text.append(word)
 return " ".join(corrected_text)
 df["text"] = df["text"].apply(lambda text: chat_words_conversion(text))<predict_on_test>
rf_model_tuned = RandomForestRegressor(bootstrap=True, ccp_alpha=0.0, criterion='mse',
max_depth=None, max_features='auto', max_leaf_nodes=None,
max_samples=None, min_impurity_decrease=0.0,
min_impurity_split=None, min_samples_leaf=1,
min_samples_split=2, min_weight_fraction_leaf=0.0,
n_estimators=2000, n_jobs=None, oob_score=False,
random_state=None, verbose=0, warm_start=False)
 rf_model_tuned.fit(train_data, y_train )
class NBSVMClassifier(BaseEstimator, ClassifierMixin):
def __init__(self, C=1.0, max_iter=100, dual=False, n_jobs=1):
self.C = C
self.dual = dual
self.n_jobs = n_jobs
self.max_iter = max_iter
def predict(self, x):
check_is_fitted(self, ['_r', '_clf'])
 return self._clf.predict(x.multiply(self._r))
def fit(self, x, y):
y = y.values
x, y = check_X_y(x, y, accept_sparse=True)
 def pr(x, y_i, y):
p = x[y==y_i].sum(0)
 return(p+1)/(( y==y_i ).sum() +1)
 self._r = sparse.csr_matrix(np.log(pr(x,1,y)/ pr(x,0,y)))
 x_nb = x.multiply(self._r)
 self._clf = LogisticRegression(C=self.C, dual=self.dual,
max_iter=self.max_iter,
n_jobs=self.n_jobs ).fit(x_nb, y)
 return self<split>
rf_model_tuned.score(valid_data, y_valid )
X_train, X_valid, y_train, y_valid = train_test_split(train["text"], train["target"],
test_size=0.2, random_state=SEED,
stratify=train["target"] )<choose_model_class>
params_for_gb_model = {
'loss': ['huber'],
'n_estimators': [500, 1000],
'learning_rate': [0.001, 0.005]
}
vec = TfidfVectorizer(ngram_range=(1,2), min_df=3, max_df=0.9,
strip_accents='unicode', use_idf=1,
smooth_idf=1, sublinear_tf=1 )<compute_train_metric>
random_search_gb_model = RandomizedSearchCV(
 estimator=gb_model,
param_distributions=params_for_gb_model,
scoring='neg_mean_absolute_error',
n_iter=5,
n_jobs=-1,
cv=3,
verbose=True
)
def objective(trial):
C = trial.suggest_float(name="C", low=1e-3, high=1e3, log=True)
 max_iter = trial.suggest_discrete_uniform(name="max_iter", low=50, high=500, q=50)
 nbsvm = NBSVMClassifier(C=C, max_iter=max_iter)
 train_term_doc = vec.fit_transform(X_train)
 valid_term_doc = vec.transform(X_valid)
 nbsvm.fit(train_term_doc, y_train)
 preds = nbsvm.predict(valid_term_doc)
 preds[preds>=0.5] = 1
preds[preds<0.5] = 0
acc = accuracy_score(y_valid, preds)
 return acc<train_model>
random_search_gb_model.fit(train_data, y_train )
study = optuna.create_study(direction="maximize")
 study.optimize(objective, n_trials=NUM_TRIALS, show_progress_bar=True )<train_model>
-1*random_search_gb_model.best_score_
print(f"Best Value: {study.best_trial.value}")
 print(f"Best Params: {study.best_params}" )<find_best_params>
random_search_gb_model.best_estimator_
kwargs = study.best_params<filter>
gb_model_tuned = GradientBoostingRegressor(alpha=0.9, ccp_alpha=0.0, criterion='friedman_mse',
init=None, learning_rate=0.005, loss='huber',
max_depth=3, max_features=None, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=1000,
n_iter_no_change=None, presort='deprecated',
random_state=None, subsample=1.0, tol=0.0001,
validation_fraction=0.1, verbose=0, warm_start=False)
 gb_model_tuned.fit(train_data, y_train )
train = df[df['target']!=-1]
test = df[df['target']==-1]<compute_test_metric>
gb_model_tuned.score(valid_data, y_valid )
def print_metrics(y_true, y_pred):
print(f"Accuracy: {accuracy_score(y_true, y_pred)}")
 print(f"MCC: {matthews_corrcoef(y_true, y_pred)}")
 print(f"F1: {f1_score(y_true, y_pred)}
" )<prepare_x_and_y>
params_for_et_model = {
'n_estimators': [100, 500, 1000],
'max_depth': [8, 16, 32, 64]
}
final_preds = np.zeros(( len(test)))
 kfold = StratifiedKFold(n_splits=NUM_SPLITS, shuffle=True, random_state=SEED)
 for fold,(train_index, valid_index)in enumerate(kfold.split(train["text"], train["target"])) :
print("*"*60)
 print("*"+" "*26+f"FOLD {fold+1}"+" "*26+"*")
 print("*"*60, end="
")
 X_train = train.iloc[train_index, :].reset_index(drop=True)
 X_valid = train.iloc[valid_index, :].reset_index(drop=True)
 y_train = X_train['target']
y_valid = X_valid['target']
train_term_doc = vec.fit_transform(X_train["text"])
 valid_term_doc = vec.transform(X_valid["text"])
 test_term_doc = vec.transform(test["text"])
 nbsvm = NBSVMClassifier(**kwargs)
 nbsvm.fit(train_term_doc, y_train)
 valid_preds = nbsvm.predict(valid_term_doc)
 print_metrics(y_valid, valid_preds)
 test_preds = nbsvm.predict(test_term_doc)
 final_preds += test_preds<load_from_csv>
random_search_et_model = RandomizedSearchCV(
 estimator=et_model,
param_distributions=params_for_et_model,
scoring='neg_mean_absolute_error',
n_iter=5,
n_jobs=-1,
cv=3,
verbose=True
)
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )<feature_engineering>
random_search_et_model.fit(train_data, y_train )
submission["target"] = final_preds/NUM_SPLITS
submission["target"] = submission["target"].apply(lambda x: 1 if x>=0.5 else 0 )<save_to_csv>
-1*random_search_et_model.best_score_
submission.to_csv("submission.csv", index=False )<save_to_csv>
random_search_et_model.best_estimator_
submission.to_csv("submission.csv", index=False )<load_from_csv>
et_model_tuned = ExtraTreesRegressor(bootstrap=False, ccp_alpha=0.0, criterion='mse',
max_depth=32, max_features='auto', max_leaf_nodes=None,
max_samples=None, min_impurity_decrease=0.0,
min_impurity_split=None, min_samples_leaf=1,
min_samples_split=2, min_weight_fraction_leaf=0.0,
n_estimators=500, n_jobs=None, oob_score=False,
random_state=None, verbose=0, warm_start=False)
 et_model_tuned.fit(train_data, y_train )
!pip install --upgrade transformers simpletransformers
train_data = pd.read_csv('.. /input/nlp-with-disaster-tweets-cleaning-data/train_data_cleaning.csv')[['text', 'target']]
test_data = pd.read_csv('.. /input/nlp-with-disaster-tweets-cleaning-data/test_data_cleaning.csv')[['id','text']]
model = ClassificationModel('distilbert', 'distilbert-base-uncased', args={'fp16': False,'train_batch_size': 4, 'gradient_accumulation_steps': 2,
'learning_rate': 1e-05, 'do_lower_case': True, 'overwrite_output_dir': True, 'manual_seed': 100, 'num_train_epochs': 2}, weight = [0.44, 0.56])
 model.train_model(train_data)
 test_data["target"], _ = model.predict(test_data['text'])
 test_data.drop(columns=['text'] ).to_csv("submission.csv", index=False )<install_modules>
et_model_tuned.score(valid_data, y_valid )
!pip install tweet-preprocessor<import_modules>
def averagePreds(validation_data):
preds_et = et_model_tuned.predict(validation_data)
 preds_xgb = xgb_model_tuned.predict(validation_data)
 preds_ada = ada_boost_tuned.predict(validation_data)
 final_preds =(preds_ada*0.6 + preds_xgb*0.2 + preds_et*0.2)
 return final_preds
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<define_variables>
valid_preds = averagePreds(valid_data)
 print("MAE of the Ensemble: ",mean_absolute_error(valid_preds, y_valid))
tf.random.set_seed(123)
 np.random.seed(123 )<define_variables>
print("AdaBoost MAE : ",mean_absolute_error(ada_boost_tuned.predict(valid_data), y_valid))
print("XGBoost MAE : ",mean_absolute_error(xgb_model_tuned.predict(valid_data), y_valid))
print("LightGBM MAE : ",mean_absolute_error(lgb_model_tuned.predict(valid_data), y_valid))
print("Random Forests MAE : ",mean_absolute_error(rf_model_tuned.predict(valid_data), y_valid))
print("Gradient Boosting MAE: ",mean_absolute_error(gb_model_tuned.predict(valid_data), y_valid))
print("Extra Trees MAE : ",mean_absolute_error(et_model_tuned.predict(valid_data), y_valid))
<load_from_csv><EOS> 
test_preds = averagePreds(test_data)
 output = pd.DataFrame({'Id':test_data.index, 'SalePrice':test_preds})
 output.to_csv('submission.csv', index=False )