Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<categorify>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
%matplotlib inline
c=['last_world','last_title','world','title']
cat_feat=[X_train.columns.get_loc(col)for col in c]
X_train=X_train.join(pd.get_dummies(X_train[c])).drop(c,axis=1 )<merge>
training_set = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 testing_set = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 sample_submission = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv" )
X_test=X_test.join(pd.get_dummies(X_test[c])).drop(c,axis=1 )<compute_train_metric>
print("Training Set: ",training_set.shape)
 print("Testing Set ",testing_set.shape )
reg_cv=cv(X_train,y_train,0,5 )<train_model>
with open("/kaggle/input/house-prices-advanced-regression-techniques/data_description.txt","r")as data_desc:
for line in data_desc.readlines() [:10]:
print(line )
reg_model,coeff=fit(X_train,y_train,0 )<predict_on_test>
y = training_set["SalePrice"]
f_test['accuracy_group']=[regression_class(pred_i,coeff)for pred_i in reg_model.predict(X_test.drop('installation_id',axis=1)) ]<define_variables>
def check_null(df,col):
return print(col," ",df[col].isnull().sum() )
Submission_file=f_test[['installation_id','accuracy_group']]<count_values>
def categorical_saleprice(df,col):
return df.groupby(col)["SalePrice"].mean()
Submission_file['accuracy_group'].value_counts() 
<save_to_csv>
check_null(training_set,columns_impute_train[0])
 categorical_saleprice(training_set,columns_impute_train[0] )
Submission_file.to_csv('submission.csv',index=False )<set_options>
MasVnrType = list(training_set.groupby(columns_impute_train[0])["SalePrice"].mean().index)
 sale_price = list(training_set.groupby(columns_impute_train[0])["SalePrice"].mean().values )
%matplotlib inline 
<load_from_csv>
training_set[training_set[columns_impute_train[0]].isnull() ]
data_dir = '.. /input/'
train = pd.read_csv(data_dir + 'train.csv')
 test = pd.read_csv(data_dir + 'test.csv' )<string_transform>
training_set.loc[training_set['Id']==235,'MasVnrType'] = "BrkFace"
training_set.loc[training_set['Id']==530,'MasVnrType'] = "BrkFace"
training_set.loc[training_set['Id']==651,'MasVnrType'] = "BrkFace"
training_set.loc[training_set['Id']==937,'MasVnrType'] = "None"
training_set.loc[training_set['Id']==974,'MasVnrType'] = "None"
training_set.loc[training_set['Id']==978,'MasVnrType'] = "BrkFace"
training_set.loc[training_set['Id']==1244,'MasVnrType'] = "Stone"
training_set.loc[training_set['Id']==1279,'MasVnrType'] = "Stone"
all_words = set([])
 train_words = []
test_words = []
train_words_len = []
test_words_len = []
regex = re.compile('[ ]')
 def preprocess_sentence(sentence):
punctuations = [',', '.', '"',<feature_engineering>
print(training_set.groupby(columns_impute_train[1])["SalePrice"].mean())
 print(training_set.groupby(columns_impute_train[0])["MasVnrArea"].value_counts())
 print(training_set[(training_set["MasVnrType"]=="Stone")&(training_set["SalePrice"] >= 230000)][["MasVnrArea","SalePrice"]].mean() )
train['word_len'] = train_words_len
test['word_len'] = test_words_len<define_variables>
training_set[(training_set["MasVnrType"]=="BrkFace")&(training_set["SalePrice"] >= 200000)][["MasVnrArea","SalePrice"]].mean()
feature_len = 150<define_variables>
training_set[training_set[columns_impute_train[1]].isnull() ][["MasVnrType","SalePrice"]]
vocab_size = len(all_words)+ 1<define_variables>
training_set.loc[training_set['Id']==937,'MasVnrArea'] = 0.0
training_set.loc[training_set['Id']==974,'MasVnrArea'] = 0.0
training_set.loc[training_set['Id']==978,'MasVnrArea'] = 332.00
training_set.loc[training_set['Id']==1244,'MasVnrArea'] = 280.3
training_set.loc[training_set['Id']==1279,'MasVnrArea'] = 280.3
training_set.loc[training_set['Id']==235,'MasVnrArea'] = 332.00
training_set.loc[training_set['Id']==530,'MasVnrArea'] = 332.00
training_set.loc[training_set['Id']==651,'MasVnrArea'] = 332.00
word2ids = {}
for index, word in enumerate(all_words):
word2ids[word] = index + 1<prepare_x_and_y>
training_set[columns_impute_train[2]].value_counts()
x_train = []
x_test = []
y_train = []
for index, row in train.iterrows() :
x = np.zeros(feature_len)
 for ix, word in enumerate(train_words[index]):
if(ix >= feature_len):
break
x[ix] = word2ids[word]
x_train.append(x)
 y = [0, 0, 0]
if row['author']=='EAP':
y[0] = 1
elif row['author']=='HPL':
y[1] = 1
else:
y[2] = 1
y_train.append(y)
 for index, row in test.iterrows() :
x = np.zeros(feature_len)
 for ix, word in enumerate(test_words[index]):
if(ix >= feature_len):
break
x[ix] = word2ids[word]
x_test.append(x)
<prepare_x_and_y>
print(training_set.groupby(columns_impute_train[2])["SalePrice"].mean() )
msk = np.random.rand(len(x_train)) < 0.95
x_train = np.array(x_train)
 y_train = np.array(y_train)
 x_valid = x_train[~msk]
y_valid = y_train[~msk]
x_train = x_train[msk]
y_train = y_train[msk]<import_modules>
def impute_BsmtQual_train(df,column):
df.loc[(df[column].isnull())&(df["SalePrice"] < 120000.00),column] = "Fa"
df.loc[(df[column].isnull())&(df["SalePrice"] >= 120000.00)&(df["SalePrice"] < 180000.00),column] = "TA"
df.loc[(df[column].isnull())&(df["SalePrice"] >= 180000.00),column] = "TA"
return "Basement Quality's missing value is imputed"
import keras
from keras.layers import *
from keras.layers.embeddings import Embedding
from keras import regularizers
from keras.models import Model
from keras.optimizers import SGD, RMSprop, Adam
from keras.models import Sequential
from keras.layers import Dense, Activation<import_modules>
impute_BsmtQual_train(training_set,"BsmtQual" )
import gensim
from gensim.models import *<define_variables>
training_set[columns_impute_train[3]].value_counts()
n_fact = 30<concatenate>
print(training_set.groupby(columns_impute_train[3])["SalePrice"].mean() )
all_sentences = train_words + test_words
word2vec = Word2Vec(all_sentences, size=n_fact, window=7, min_count=2, workers=4, iter=20 )<feature_engineering>
training_set.loc[(training_set[columns_impute_train[3]].isnull())&(training_set["SalePrice"] < 75000.00),"BsmtCond"] = "Po"
training_set.loc[(training_set[columns_impute_train[3]].isnull())&(training_set["SalePrice"] >= 75000.00)&(training_set["SalePrice"] < 140000.00),"BsmtCond"] = "Fa"
training_set.loc[(training_set[columns_impute_train[3]].isnull())&(training_set["SalePrice"] >= 140000.00)&(training_set["SalePrice"] < 190000.00),"BsmtCond"] = "TA"
training_set.loc[(training_set[columns_impute_train[3]].isnull())&(training_set["SalePrice"] >= 190000.00),"BsmtCond"] = "Gd"
emb = np.zeros(( vocab_size, n_fact))
for word,idx in word2ids.items() :
print(word)
 if word in word2vec.wv:
emb[idx] = word2vec.wv[word]
else:
print("word is not present:" + str(word))
emb[idx] = normal(scale=0.6, size=(n_fact,))<choose_model_class>
training_set[columns_impute_train[4]].value_counts()
model = Sequential([
Embedding(vocab_size, n_fact, input_length=feature_len, weights=[emb], trainable=True),
Dropout(0.5),
Conv1D(30, 7, border_mode='same', activation='relu'),
Dropout(0.5),
Flatten() ,
BatchNormalization() ,
Dense(40, activation='relu'),
Dropout(0.6),
Dense(3, activation='softmax')] )<choose_model_class>
print(training_set.groupby(columns_impute_train[4])["SalePrice"].mean() )
model.compile(loss='categorical_crossentropy', optimizer=Adam() , metrics=['accuracy'])
 model.summary()<train_model>
training_set.loc[(training_set[columns_impute_train[4]].isnull())&(training_set["SalePrice"] < 150000.00),"BsmtExposure"] = "No"
training_set.loc[(training_set[columns_impute_train[4]].isnull())&(training_set["SalePrice"] >= 150000.00)&(training_set["SalePrice"] < 200000.00),"BsmtExposure"] = "Mn"
training_set.loc[(training_set[columns_impute_train[4]].isnull())&(training_set["SalePrice"] >= 200000.00)&(training_set["SalePrice"] < 230000.00),"BsmtExposure"] = "Av"
training_set.loc[(training_set[columns_impute_train[4]].isnull())&(training_set["SalePrice"] >= 210000.00),"BsmtExposure"] = "Gd"
model.fit(np.array(x_train), y_train, validation_data=(np.array(x_valid),y_valid), nb_epoch=3, batch_size=64 )<train_model>
training_set[columns_impute_train[5]].value_counts()
model.fit(np.array(x_train), y_train, validation_data=(np.array(x_valid),y_valid), nb_epoch=3, batch_size=64 )<predict_on_test>
print(training_set.groupby(columns_impute_train[5])["SalePrice"].mean() )
pred = model.predict(np.array(x_test))<save_to_csv>
training_set.loc[(training_set[columns_impute_train[5]].isnull())&(training_set["BsmtQual"] == "Fa"),"BsmtFinType1"] = "Unf"
training_set.loc[(training_set[columns_impute_train[5]].isnull())&(training_set["BsmtQual"] == "TA"),"BsmtFinType1"] = "LwQ"
training_set.loc[(training_set[columns_impute_train[5]].isnull())&(training_set["BsmtQual"] == "Gd"),"BsmtFinType1"] = "GLQ"
submission = pd.DataFrame()
submission ['id'] = test['id']
submission ['EAP'] = pred[:,0]
submission ['HPL'] = pred[:,1]
submission ['MWS'] = pred[:,2]
submission.to_csv('sub.csv', index=False )<set_options>
training_set[columns_impute_train[6]].value_counts()
%matplotlib inline
np.random.seed(25)
 print(check_output(["ls", ".. /input"] ).decode("utf8"))
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
<categorify>
print(training_set.groupby(columns_impute_train[6])["SalePrice"].mean() )
mapping_target = {'EAP':0, 'HPL':1, 'MWS':2}
train = train.replace({'author':mapping_target} )<prepare_x_and_y>
training_set[training_set["BsmtQual"]=="Ex"]["BsmtFinType2"].value_counts()
test_id = test['id']
target = train['author']<string_transform>
training_set.loc[training_set[columns_impute_train[6]].isnull() ,"BsmtFinType2"] = "Unf"
def preprocess(text):
text = text.strip()
text = text.replace("' ", " ' ")
 signs = set(',.:;"?!')
 prods = set(text)& signs
if not prods:
return text
for sign in prods:
text = text.replace(sign, ' {} '.format(sign))
return text
def cleanData(text, lowercase = False, remove_stops = False, stemming = False, lemmatization = False):
txt = str(text)
 txt = re.sub(r'[^A-Za-z\s]',r' ',txt)
 if lowercase:
txt = " ".join([w.lower() for w in txt.split() ])
 if remove_stops:
txt = " ".join([w for w in txt.split() if w not in stops])
 if stemming:
st = PorterStemmer()
txt = " ".join([st.stem(w)for w in txt.split() ])
 if lemmatization:
wordnet_lemmatizer = WordNetLemmatizer()
txt = " ".join([wordnet_lemmatizer.lemmatize(w, pos='v')for w in txt.split() ])
 return txt<categorify>
training_set[columns_impute_train[7]].value_counts()
train['text'] = train['text'].map(lambda x: preprocess(x))
test['text'] = test['text'].map(lambda x: preprocess(x))<categorify>
training_set[columns_impute_train[7]].fillna(value="SBrkr",inplace=True )
train['text'] = train['text'].map(lambda x: cleanData(x, lowercase=True, remove_stops=False, stemming=False, lemmatization = False))
test['text'] = test['text'].map(lambda x: cleanData(x, lowercase=True, remove_stops=False, stemming=False, lemmatization = False))<import_modules>
training_set[columns_impute_train[8]].value_counts()
np.random.seed(25)
<define_variables>
training_set[training_set[columns_impute_train[8]].isnull() ]
MAX_SEQUENCE_LENGTH = 256
MAX_NB_WORDS = 200000<statistical_test>
print(training_set.groupby(columns_impute_train[8])["SalePrice"].mean() )
def add_ngram(q, n_gram_max):
ngrams = []
for n in range(2, n_gram_max+1):
for w_index in range(len(q)-n+1):
ngrams.append('--'.join(q[w_index:w_index+n]))
return q + ngrams<string_transform>
training_set.loc[(training_set[columns_impute_train[8]].isnull())&(training_set["SalePrice"] < 110000.00),"GarageType"] = "CarPort"
training_set.loc[(training_set[columns_impute_train[8]].isnull())&(training_set["SalePrice"] >= 110000.00)&(training_set["SalePrice"] < 140000.00),"GarageType"] = "Detchd"
training_set.loc[(training_set[columns_impute_train[8]].isnull())&(training_set["SalePrice"] >= 140000.00)&(training_set["SalePrice"] < 210000.00),"GarageType"] = "Attchd"
training_set.loc[(training_set[columns_impute_train[8]].isnull())&(training_set["SalePrice"] >= 210000.00),"GarageType"] = "BuiltIn"
n_gram_max = 2
print('Processing text dataset')
 texts_1 = []
for text in train['text']:
text = text.split()
texts_1.append(' '.join(add_ngram(text, n_gram_max)))
 labels = train['author']
print('Found %s texts.' % len(texts_1))
test_texts_1 = []
for text in test['text']:
text = text.split()
test_texts_1.append(' '.join(add_ngram(text, n_gram_max)))
 print('Found %s texts.' % len(test_texts_1))<string_transform>
columns_impute_train.drop(columns_impute_train[9] )
min_count = 2
tokenizer = Tokenizer(lower=False, filters='')
 tokenizer.fit_on_texts(texts_1)
 num_words = sum([1 for _, v in tokenizer.word_counts.items() if v >= min_count])
 tokenizer = Tokenizer(num_words=num_words, lower=True, filters='')
 tokenizer.fit_on_texts(texts_1)
 sequences_1 = tokenizer.texts_to_sequences(texts_1)
 test_sequences_1 = tokenizer.texts_to_sequences(test_texts_1)
 data_1 = pad_sequences(sequences_1, maxlen=MAX_SEQUENCE_LENGTH)
 labels = np.array(labels)
 print('Shape of data tensor:', data_1.shape)
 print('Shape of label tensor:', labels.shape)
 test_data_1 = pad_sequences(test_sequences_1, maxlen=MAX_SEQUENCE_LENGTH)
 del test_sequences_1
del sequences_1
gc.collect()<define_variables>
training_set[columns_impute_train[10]].value_counts()
nb_words = np.max(data_1)+ 1<choose_model_class>
training_set[training_set[columns_impute_train[10]].isnull() ]
model = Sequential()
model.add(Embedding(nb_words,20,input_length=MAX_SEQUENCE_LENGTH))
model.add(GlobalAveragePooling1D())
 model.add(Dense(3, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics = ['accuracy'] )<train_model>
print(training_set.groupby(columns_impute_train[10])["SalePrice"].mean() )
model.fit(data_1, to_categorical(labels), validation_split=0.2, nb_epoch=15, batch_size=16 )<predict_on_test>
training_set.loc[(training_set[columns_impute_train[10]].isnull())&(training_set["SalePrice"] < 160000.00),"GarageFinish"] = "Unf"
training_set.loc[(training_set[columns_impute_train[10]].isnull())&(training_set["SalePrice"] >= 160000.00)&(training_set["SalePrice"] < 220000.00),"GarageFinish"] = "RFn"
training_set.loc[(training_set[columns_impute_train[10]].isnull())&(training_set["SalePrice"] >= 220000.00),"GarageFinish"] = "Fin"
preds = model.predict(test_data_1 )<save_to_csv>
training_set[columns_impute_train[11]].value_counts()
result = pd.DataFrame()
result['id'] = test_id
result['EAP'] = [x[0] for x in preds]
result['HPL'] = [x[1] for x in preds]
result['MWS'] = [x[2] for x in preds]
result.to_csv("result.csv", index=False )<set_options>
print(training_set.groupby(columns_impute_train[11])["SalePrice"].mean() )
%config InlineBackend.figure_format = 'svg'
%matplotlib inline
sns.set()
pd.options.display.max_colwidth = 160<load_from_csv>
training_set.loc[(training_set[columns_impute_train[11]].isnull())&(training_set["SalePrice"] < 100000.00),"GarageQual"] = "Po"
training_set.loc[(training_set[columns_impute_train[11]].isnull())&(training_set["SalePrice"] >= 100000.00)&(training_set["SalePrice"] < 140000.00),"GarageQual"] = "Fa"
training_set.loc[(training_set[columns_impute_train[11]].isnull())&(training_set["SalePrice"] >= 140000.00)&(training_set["SalePrice"] < 220000.00),"GarageQual"] = "Ta"
training_set.loc[(training_set[columns_impute_train[11]].isnull())&(training_set["SalePrice"] >= 220000.00),"GarageQual"] = "Ex"
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<import_modules>
training_set[columns_impute_train[12]].value_counts()
import spacy
import nltk<feature_engineering>
print(training_set.groupby(columns_impute_train[12])["SalePrice"].mean() )
nlp = spacy.en.English()<string_transform>
training_set[columns_impute_train[12]].fillna(value="TA",inplace=True )
spacy.en.STOP_WORDS.add("'s")
 for word in spacy.en.STOP_WORDS:
lexeme = nlp.vocab[word]
lexeme.is_stop = True<categorify>
training_set.drop(columns=columns_too_many_missing,inplace=True )
def replace_ents(doc):
prefix = 'ent__'
text = str(doc.doc)
 for ent in doc.ents:
text = text.replace(ent.orth_, prefix + ent.label_)
 return text<feature_engineering>
testing_set.drop(columns=columns_too_many_missing,inplace=True )
def preprocess(df):
print('Started parsing...')
 doc = df.text.apply(nlp)
 print('Text parsed')
 df['n_char'] = df.text.apply(len)
 df['n_words'] = doc.apply(lambda x: len([t for t in x if not t.is_punct]))
df['n_punct'] = doc.apply(lambda x: len([t for t in x if t.is_punct]))
df['n_ents'] = doc.apply(lambda x: len(x.ents))
df['n_chunks'] = doc.apply(lambda x: len(list(x.noun_chunks)))
 df['n_unique_words'] = doc.apply(lambda x: len(set([t.lower_ for t in x if not t.is_punct])))
 df['n_stop_words'] = doc.apply(lambda x: len([t for t in x if t.is_stop]))
df['char_by_word'] = doc.apply(lambda x: np.mean([len(t.orth_)for t in x if not t.is_punct]))
print('Features created')
 df['text_ent_repl'] = doc.apply(replace_ents)
 print('Entities replaced')
 clean_and_lemmatize = lambda x: ' '.join([t.lemma_ for t in x if not t.is_punct and not t.is_stop])
 df['text_cleaned'] = doc.apply(clean_and_lemmatize)
 print('Text cleaned' )<feature_engineering>
training_set.select_dtypes(include='object' ).columns
col_to_encode = ['MSZoning', 'Street', 'LotShape', 'LandContour',
'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2',
'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st',
'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual',
'Functional', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond',
'PavedDrive', 'SaleType', 'SaleCondition']
eap = train.loc[train.author == 'EAP']
hpl = train.loc[train.author == 'HPL']
mws = train.loc[train.author == 'MWS']
eap_t = eap.text_cleaned
hpl_t = hpl.text_cleaned
mws_t = mws.text_cleaned<string_transform>
encoded_categorical_df = pd.get_dummies(training_set, columns = col_to_encode, drop_first = True )
fd = nltk.FreqDist([y for x in train.text_cleaned.str.split() for y in x])
 fd_eap = nltk.FreqDist([y for x in eap_t.str.split() for y in x])
 fd_hpl = nltk.FreqDist([y for x in hpl_t.str.split() for y in x])
 fd_mws = nltk.FreqDist([y for x in mws_t.str.split() for y in x] )<set_options>
col_drop = list(training_set.select_dtypes(include='object' ).columns)
 col_drop.append("Id")
 training_set.drop(columns=col_drop,inplace=True )
sns.set_style({'axes.grid' : False} )<data_type_conversions>
encoded_categorical_df.drop(columns=["Id","Utilities"],inplace = True )
WordCloud(min_font_size=8, width=1000, height=400, colormap='cubehelix_r' ).generate(' '.join(hpl_t)).to_image()<categorify>
min_max_scaler = MinMaxScaler()
standard_scaler = StandardScaler()
col_to_normalize = ['MSSubClass', 'LotArea', 'MasVnrArea', 'BsmtFinSF1',
'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF','LowQualFinSF', 'GrLivArea'
,'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch']
encoded_categorical_df[col_to_normalize] = min_max_scaler.fit_transform(encoded_categorical_df[col_to_normalize])
WordCloud(min_font_size=8, width=1000, height=400, colormap='hot' ).generate(' '.join(mws_t)).to_image()<set_options>
encoded_categorical_df[['MSSubClass', 'LotArea', 'MasVnrArea', 'BsmtFinSF1',
'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF','LowQualFinSF', 'GrLivArea'
,'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch']]
sns.set()<filter>
encoded_categorical_df.drop(columns=["SalePrice"],inplace=True )
def drop_outliers(s):
med = s.mean()
std = s.std()
return s[(med - 3*std <= s)&(s <= med + 3*std)]<import_modules>
testing_set["MSZoning"].value_counts()
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline<prepare_x_and_y>
testing_set["MSSubClass"].value_counts()
y = train.author<feature_engineering>
testing_set[testing_set[columns_impute_test[0]].isnull() ]
vectorizer = CountVectorizer(
 token_pattern=r'\w{1,}',
ngram_range=(1, 2), stop_words='english' 
)
 X = vectorizer.fit_transform(train.text )<compute_train_metric>
testing_set[testing_set[columns_impute_test[0]].isnull() ][["MSSubClass","MSZoning"]]
%%time
scores = cross_val_score(LogisticRegression() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )<compute_train_metric>
testing_set.loc[(testing_set[columns_impute_test[0]].isnull())&(testing_set["MSSubClass"] == 20),"MSZoning"] = "RL"
testing_set.loc[(testing_set[columns_impute_test[0]].isnull())&(testing_set["MSSubClass"] == 30),"MSZoning"] = "RM"
testing_set.loc[(testing_set[columns_impute_test[0]].isnull())&(testing_set["MSSubClass"] == 70),"MSZoning"] = "RM"
%%time
scores = cross_val_score(MultinomialNB() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )<feature_engineering>
testing_set[columns_impute_test[1]].value_counts()
vectorizer = CountVectorizer(ngram_range=(1,2))
X = vectorizer.fit_transform(train.text_cleaned )<compute_train_metric>
testing_set[testing_set[columns_impute_test[1]].isnull() ]
%%time
scores = cross_val_score(LogisticRegression() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )<compute_train_metric>
testing_set[columns_impute_test[1]].fillna(value="AllPub",inplace=True )
%%time
scores = cross_val_score(MultinomialNB() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )<feature_engineering>
testing_set[columns_impute_test[2]].value_counts()
vectorizer = CountVectorizer(token_pattern=r'\w{1,}', stop_words='english', ngram_range=(1,2))
X = vectorizer.fit_transform(train.text_ent_repl )<compute_train_metric>
testing_set[testing_set[columns_impute_test[2]].isnull() ]
%%time
scores = cross_val_score(LogisticRegression() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )<compute_train_metric>
testing_set[columns_impute_test[2]].fillna(value="VinylSd",inplace=True )
%%time
scores = cross_val_score(MultinomialNB() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )<feature_engineering>
testing_set[columns_impute_test[3]].value_counts()
vectorizer = TfidfVectorizer(
 token_pattern=r'\w{1,}', ngram_range=(1, 1),
use_idf=True, smooth_idf=True, sublinear_tf=True, 
)
 X = vectorizer.fit_transform(train.text_cleaned )<compute_train_metric>
testing_set[testing_set[columns_impute_test[3]].isnull() ]
%%time
scores = cross_val_score(LogisticRegression() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )<compute_train_metric>
testing_set[columns_impute_test[3]].fillna(value="VinylSd",inplace=True )
%%time
scores = cross_val_score(MultinomialNB() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )<import_modules>
testing_set[columns_impute_test[4]].value_counts()
import xgboost as xgb
import lightgbm as lgb<define_variables>
testing_set[testing_set[columns_impute_test[4]].isnull() ][["MasVnrType","ExterCond","ExterQual"]]
drop = ['id', 'text', 'text_cleaned', 'text_ent_repl']<drop_column>
testing_set[testing_set["ExterQual"]=="Gd"]["MasVnrType"].value_counts()
X_meta = train.drop(drop + ['author'], axis=1 )<compute_train_metric>
testing_set["MasVnrType"].fillna(value="None",inplace=True )
%%time
lgbc = lgb.LGBMClassifier(objective='multiclass', n_estimators=100)
 scores = cross_val_score(lgbc, X_meta, y, cv=4, n_jobs=1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )<compute_train_metric>
testing_set[columns_impute_test[5]].value_counts()
%%time
xgbc = xgb.XGBClassifier(objective='multi:softprob', n_estimators=200)
 scores = cross_val_score(xgbc, X_meta, y, cv=4, n_jobs=4, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )<train_model>
testing_set[testing_set[columns_impute_test[5]].isnull() ]
xgbc.fit(X_meta, y);
xgb.plot_importance(xgbc);<import_modules>
testing_set[columns_impute_test[5]].fillna(value=0.0,inplace=True )
from sklearn.model_selection import StratifiedKFold
from sklearn import metrics<feature_engineering>
testing_set[columns_impute_test[6]].value_counts()
def add_prob_features(vectorizer, col, model, prefix, cv=5):
vectorizer.fit(train[col].append(test[col]))
X = vectorizer.transform(train[col])
 X_test = vectorizer.transform(test[col])
 cv_scores = []
pred_test = 0
pred_train = np.zeros([train.shape[0], 3])
 kf = StratifiedKFold(n_splits=cv, shuffle=True, random_state=123)
 print('CV started')
 for train_index, dev_index in kf.split(X, y):
X_train, X_dev = X[train_index], X[dev_index]
y_train, y_dev = y[train_index], y[dev_index]
model.fit(X_train, y_train)
 pred_dev = model.predict_proba(X_dev)
 pred_test += model.predict_proba(X_test)
 pred_train[dev_index, :] = pred_dev
cv_scores.append(metrics.log_loss(y_dev, pred_dev))
print('.', end='')
 print('')
 print("Mean CV LogLoss: %.3f" %(np.mean(cv_scores)))
 pred_test /= cv
train[prefix+'eap'] = pred_train[:, 0]
train[prefix+'hpl'] = pred_train[:, 1]
train[prefix+'mws'] = pred_train[:, 2]
test[prefix+'eap'] = pred_test[:, 0]
test[prefix+'hpl'] = pred_test[:, 1]
test[prefix+'mws'] = pred_test[:, 2]<feature_engineering>
testing_set.loc[(testing_set[columns_impute_test[6]].isnull())&(testing_set["GarageQual"] == "Ta"),"BsmtQual"] = "Gd"
testing_set[columns_impute_test[6]].fillna(value="TA",inplace=True )
vectorizer = CountVectorizer(
 token_pattern=r'\w{1,}',
ngram_range=(1, 2), stop_words='english' 
)
 add_prob_features(vectorizer, 'text', MultinomialNB() , 'nb_ctv_', cv=40 )<feature_engineering>
testing_set[columns_impute_test[7]].value_counts()
vectorizer = TfidfVectorizer(
 token_pattern=r'\w{1,}', ngram_range=(1, 1),
use_idf=True, smooth_idf=True, sublinear_tf=True, 
)
 add_prob_features(vectorizer, 'text_cleaned', MultinomialNB() , 'nb_tfv_', cv=40 )<feature_engineering>
testing_set[columns_impute_test[7]].fillna(value="TA",inplace=True )
vectorizer = TfidfVectorizer(
 ngram_range=(1, 5), analyzer='char' 
)
 add_prob_features(vectorizer, 'text', MultinomialNB() , 'nb_char_', cv=40 )<feature_engineering>
testing_set[columns_impute_test[8]].value_counts()
vectorizer = CountVectorizer(
 token_pattern=r'\w{1,}',
ngram_range=(1, 2), stop_words='english' 
)
 add_prob_features(vectorizer, 'text_ent_repl', MultinomialNB() , 'nb_ent_', cv=40 )<drop_column>
testing_set[testing_set["GarageQual"]=="TA"]["BsmtExposure"].value_counts()
X = train.drop(drop + ['author'], axis=1 )<compute_train_metric>
testing_set[columns_impute_test[8]].fillna(value="No",inplace=True )
%%time
lgbc = lgb.LGBMClassifier(objective='multiclass', n_estimators=150, num_leaves=10)
 scores = cross_val_score(lgbc, X, y, cv=4, n_jobs=1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )<compute_train_metric>
testing_set[columns_impute_test[9]].value_counts()
%%time
xgbc = xgb.XGBClassifier(objective='multi:softprob', n_estimators=150)
 scores = cross_val_score(xgbc, X, y, cv=4, n_jobs=1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )<train_model>
testing_set[testing_set["BldgType"]=="1Fam"]["BsmtFinType1"].value_counts()
%%time
xgbc.fit(X, y);<save_to_csv>
testing_set.loc[(testing_set[columns_impute_test[9]].isnull())&(testing_set["BldgType"] == "1Fam"),"BsmtFinType1"] = "Unf"
testing_set.loc[(testing_set[columns_impute_test[9]].isnull())&(testing_set["BldgType"] == "Duplex"),"BsmtFinType1"] = "GLQ"
def sub(est, name='sub.csv'):
sub = pd.DataFrame(est.predict_proba(test.drop(drop, axis=1)) , columns=['EAP', 'HPL', 'MWS'])
 sub.insert(0, 'id', test.id)
 sub.to_csv(name, index=False )<import_modules>
testing_set[columns_impute_test[10]].value_counts()
from sklearn import model_selection<choose_model_class>
testing_set[testing_set[columns_impute_test[10]].isnull() ]
clf = xgb.XGBClassifier(objective = 'multi:softprob', nthread=1)
 parameters = {
'n_estimators': [150],
'max_depth': [3],
'subsample': [0.65],
'colsample_bytree': [0.95],
'min_child_weight': [1],
}
clf = model_selection.GridSearchCV(clf, parameters, n_jobs=4, verbose=1, scoring='neg_log_loss', refit=True )<train_model>
testing_set[testing_set["SaleCondition"]=="Abnorml"][columns_impute_test[10]].value_counts().head()
clf.fit(X, y);<find_best_params>
testing_set[columns_impute_test[10]].fillna(value=0.0,inplace=True )
-clf.best_score_<find_best_params>
testing_set[columns_impute_test[11]].value_counts()
sub(clf.best_estimator_ )<set_options>
testing_set[testing_set["SaleCondition"]=="AdjLand"][columns_impute_test[11]].value_counts()
%matplotlib inline<load_from_csv>
testing_set[columns_impute_test[11]].fillna(value="Unf",inplace=True)
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<count_values>
testing_set[columns_impute_test[12]].fillna(value=0.0,inplace=True )
train_df['author'].value_counts()<feature_engineering>
testing_set[columns_impute_test[13]].value_counts()
train_df['text_length'] = train_df['text'].str.len()<feature_engineering>
testing_set[testing_set[columns_impute_test[13]].isnull() ][["BsmtFinType1","GarageCond","GarageQual","BldgType"]]
test_df['text_length'] = test_df['text'].str.len()<categorify>
testing_set[columns_impute_test[13]].fillna(value=0.0,inplace=True )
train_df['author_num'] = train_df.author.map({'EAP':0, 'HPL':1, 'MWS':2})
 train_df.head()<feature_engineering>
testing_set[columns_impute_test[14]].value_counts()
train_df = train_df.rename(columns={'text':'original_text'})
 train_df['text'] = train_df['original_text'].str[:700]
train_df['text_length'] = train_df['text'].str.len()<feature_engineering>
testing_set[testing_set[columns_impute_test[14]].isnull() ][["BsmtFinType1","GarageCond","GarageQual","BldgType"]]
test_df = test_df.rename(columns={'text':'original_text'})
 test_df['text'] = test_df['original_text'].str[:700]
test_df['text_length'] = test_df['text'].str.len()<prepare_x_and_y>
testing_set[columns_impute_test[14]].fillna(value=0.0,inplace=True )
X = train_df['text']
y = train_df['author_num']<split>
testing_set[columns_impute_test[15]].value_counts()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)
 print(X_train.shape, y_train.shape, X_test.shape, y_test.shape )<count_values>
testing_set[testing_set[columns_impute_test[15]].isnull() ][["BsmtFinType1","GarageCond","GarageQual","BldgType"]]
print(y_train.value_counts() ,'
', y_test.value_counts() )<define_variables>
testing_set[columns_impute_test[15]].fillna(value=0.0,inplace=True )
vect = CountVectorizer(lowercase=False, token_pattern=r'(?u)\b\w+\b|\,|\.|\;|\:')
 vect<normalization>
testing_set[columns_impute_test[16]].value_counts()
X_train_dtm = vect.fit_transform(X_train)
 X_train_dtm<normalization>
testing_set[columns_impute_test[16]].fillna(value=0.0,inplace=True )
X_test_dtm = vect.transform(X_test)
 X_test_dtm<feature_engineering>
testing_set[columns_impute_test[17]].value_counts()
X_train_chars = X_train.str.len()
X_train_punc = X_train.apply(lambda x: len([c for c in str(x)if c in punctuation]))
X_test_chars = X_test.str.len()
X_test_punc = X_test.apply(lambda x: len([c for c in str(x)if c in punctuation]))
X_train_dtm = add_feature(X_train_dtm, [X_train_chars, X_train_punc])
 X_test_dtm = add_feature(X_test_dtm, [X_test_chars, X_test_punc] )<define_search_model>
testing_set[testing_set[columns_impute_test[17]].isnull() ][["BsmtFinType1","GarageCond","GarageQual","BldgType"]]
nb = MultinomialNB()
nb<train_on_grid>
testing_set[columns_impute_test[17]].fillna(value="Fa",inplace=True )
grid_values = {'alpha':[0.01, 0.1, 1.0, 10.0, 100.0]}
grid_nb = GridSearchCV(nb, param_grid=grid_values, scoring='neg_log_loss')
 grid_nb.fit(X_train_dtm, y_train)
 grid_nb.best_params_<train_model>
testing_set[columns_impute_test[18]].value_counts()
nb = MultinomialNB(alpha=1.0)
 nb.fit(X_train_dtm, y_train )<predict_on_test>
testing_set[testing_set[columns_impute_test[18]].isnull() ][["BsmtFinType1","GarageCond","GarageQual","BldgType"]]
y_pred_test = nb.predict(X_test_dtm )<compute_test_metric>
testing_set[columns_impute_test[18]].fillna(value="Fa",inplace=True )
metrics.accuracy_score(y_test, y_pred_test )<compute_train_metric>
testing_set[columns_impute_test[19]].value_counts()
y_pred_train = nb.predict(X_train_dtm)
 metrics.accuracy_score(y_train, y_pred_train )<compute_test_metric>
testing_set[testing_set[columns_impute_test[19]].isnull() ][["BldgType","GarageType"]]
metrics.confusion_matrix(y_test, y_pred_test )<predict_on_test>
testing_set[testing_set["BldgType"]=="TwnhsE"][columns_impute_test[19]].value_counts()
y_pred_prob = nb.predict_proba(X_test_dtm)
 y_pred_prob[:10]<compute_test_metric>
testing_set.loc[(testing_set[columns_impute_test[19]].isnull())&(testing_set["BldgType"] == "1Fam"),"GarageType"] = "Attchd"
testing_set.loc[(testing_set[columns_impute_test[19]].isnull())&(testing_set["BldgType"] == "Duplex"),"GarageType"] = "Detchd"
testing_set.loc[(testing_set[columns_impute_test[19]].isnull())&(testing_set["BldgType"] == "2fmCon"),"GarageType"] = "Detchd"
testing_set.loc[(testing_set[columns_impute_test[19]].isnull())&(testing_set["BldgType"] == "Twnhs"),"GarageType"] = "Detchd"
testing_set.loc[(testing_set[columns_impute_test[19]].isnull())&(testing_set["BldgType"] == "TwnhsE"),"GarageType"] = "Attchd"
metrics.log_loss(y_test, y_pred_prob )<choose_model_class>
testing_set[columns_impute_test[21]].value_counts()
logreg = LogisticRegression(random_state=8)
 logreg<train_on_grid>
testing_set[testing_set[columns_impute_test[21]].isnull() ]["BldgType"].value_counts()
grid_values = {'C':[0.01, 0.1, 1.0, 3.0, 5.0]}
grid_logreg = GridSearchCV(logreg, param_grid=grid_values, scoring='neg_log_loss')
 grid_logreg.fit(X_train_dtm, y_train)
 grid_logreg.best_params_<train_model>
testing_set.loc[(testing_set[columns_impute_test[21]].isnull())&(testing_set["BldgType"] == "TwnhsE"),"GarageFinish"] = "Fin"
testing_set[columns_impute_test[21]].fillna(value="Unf",inplace=True )
logreg = LogisticRegression(C=1.0, random_state=8)
 logreg.fit(X_train_dtm, y_train )<predict_on_test>
testing_set[columns_impute_test[22]].value_counts()
y_pred_test = logreg.predict(X_test_dtm )<compute_test_metric>
testing_set[testing_set[columns_impute_test[22]].isnull() ]
metrics.accuracy_score(y_test, y_pred_test )<compute_train_metric>
testing_set[testing_set["SaleCondition"]=="Alloca"][columns_impute_test[22]].value_counts()
y_pred_train = logreg.predict(X_train_dtm)
 metrics.accuracy_score(y_train, y_pred_train )<compute_test_metric>
testing_set[columns_impute_test[22]].fillna(value=2.0,inplace=True )
metrics.confusion_matrix(y_test, y_pred_test )<predict_on_test>
testing_set[columns_impute_test[23]].value_counts()
y_pred_prob = logreg.predict_proba(X_test_dtm)
 y_pred_prob[:10]<compute_test_metric>
testing_set[testing_set[columns_impute_test[23]].isnull() ]
metrics.log_loss(y_test, y_pred_prob )<feature_engineering>
testing_set[testing_set["SaleCondition"]=="Alloca"][columns_impute_test[23]].mean()
X_dtm = vect.fit_transform(X)
 X_dtm<feature_engineering>
testing_set[columns_impute_test[23]].fillna(value=499.09,inplace=True )
X_chars = X.str.len()
X_punc = X.apply(lambda x: len([c for c in str(x)if c in punctuation]))
X_dtm = add_feature(X_dtm, [X_chars, X_punc])
 X_dtm<train_model>
testing_set[columns_impute_test[24]].value_counts()
logreg.fit(X_dtm, y )<compute_train_metric>
testing_set[columns_impute_test[24]].fillna(value="TA",inplace=True )
y_pred_train = logreg.predict(X_dtm)
 metrics.accuracy_score(y, y_pred_train )<categorify>
testing_set[columns_impute_test[25]].value_counts()
test = test_df['text']
test_dtm = vect.transform(test)
 test_dtm<feature_engineering>
testing_set[columns_impute_test[25]].fillna(value="TA",inplace=True )
test_chars = test.str.len()
test_punc = test.str.count(r'\W')
 test_dtm = add_feature(test_dtm, [test_chars, test_punc])
 test_dtm<predict_on_test>
testing_set[columns_impute_test[26]].value_counts()
LR_y_pred = logreg.predict(test_dtm)
 print(LR_y_pred )<predict_on_test>
testing_set[testing_set[columns_impute_test[26]].isnull() ]
LR_y_pred_prob = logreg.predict_proba(test_dtm)
 LR_y_pred_prob[:10]<train_model>
testing_set[testing_set["SaleCondition"]=="Normal"][columns_impute_test[26]].value_counts()
nb.fit(X_dtm, y )<compute_train_metric>
testing_set[columns_impute_test[26]].fillna(value="WD",inplace=True )
y_pred_train = nb.predict(X_dtm)
 metrics.accuracy_score(y, y_pred_train )<predict_on_test>
testing_set.columns[(testing_set.isnull().sum() > 0)&(testing_set.isnull().sum() < 100)]
NB_y_pred = nb.predict(test_dtm)
 print(NB_y_pred )<predict_on_test>
testing_set.drop(columns=["Id"],inplace=True)
 testing_set.drop(columns=["Utilities"],inplace=True )
NB_y_pred_prob = nb.predict_proba(test_dtm)
 NB_y_pred_prob[:10]<prepare_output>
col_for_dummies = ["MSZoning","Street","LotShape","LandContour","LotConfig","LandSlope","Neighborhood","Condition1","Condition2","BldgType",
"HouseStyle","RoofStyle",'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'Heating', 'HeatingQC', 'CentralAir', 'Electrical',
'KitchenQual', 'Functional', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive', 'SaleType', 'SaleCondition']
alpha = 0.6
y_pred_prob =(( 1-alpha)*LR_y_pred_prob + alpha*NB_y_pred_prob)
 y_pred_prob[:10]<prepare_output>
testing_encoding_categorical = pd.get_dummies(testing_set, columns = col_for_dummies, drop_first = True )
result = pd.DataFrame(y_pred_prob, columns=['EAP','HPL','MWS'])
 result.insert(0, 'id', test_df['id'])
 result.head()<save_to_csv>
col_to_normalize = ["MSSubClass","LotArea","MasVnrArea","BsmtFinSF1","BsmtFinSF2","BsmtUnfSF","TotalBsmtSF","1stFlrSF","2ndFlrSF","LowQualFinSF",
"GrLivArea","GarageArea","WoodDeckSF","OpenPorchSF","EnclosedPorch","3SsnPorch","ScreenPorch"]
testing_encoding_categorical[col_to_normalize] = min_max_scaler.transform(testing_encoding_categorical[col_to_normalize])
result.to_csv('rhodium_submission_16.csv', index=False, float_format='%.20f' )<set_options>
features_to_delete_test = []
for col in testing_encoding_categorical.columns:
if col not in encoded_categorical_df.columns:
features_to_delete_test.append(col )
color = sns.color_palette()
%matplotlib inline
eng_stopwords = set(stopwords.words("english"))
pd.options.mode.chained_assignment = None<load_from_csv>
features_to_delete_train = []
for col in encoded_categorical_df.columns:
if col not in testing_encoding_categorical.columns:
features_to_delete_train.append(col )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Number of rows in train dataset : ",train_df.shape[0])
 print("Number of rows in test dataset : ",test_df.shape[0] )<groupby>
encoded_categorical_df.drop(columns=features_to_delete_train,inplace=True )
grouped_df = train_df.groupby('author')
 for name, group in grouped_df:
print("Author name : ", name)
 cnt = 0
for ind, row in group.iterrows() :
print(row["text"])
 cnt += 1
if cnt == 5:
break
print("
" )<feature_engineering>
testing_encoding_categorical.drop(columns=features_to_delete_test,inplace=True )
train_df["num_words"] = train_df["text"].apply(lambda x: len(str(x ).split()))
test_df["num_words"] = test_df["text"].apply(lambda x: len(str(x ).split()))
train_df["num_unique_words"] = train_df["text"].apply(lambda x: len(set(str(x ).split())))
 test_df["num_unique_words"] = test_df["text"].apply(lambda x: len(set(str(x ).split())))
 train_df["num_chars"] = train_df["text"].apply(lambda x: len(str(x)))
 test_df["num_chars"] = test_df["text"].apply(lambda x: len(str(x)))
 train_df["num_stopwords"] = train_df["text"].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
test_df["num_stopwords"] = test_df["text"].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
train_df["num_punctuations"] =train_df['text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
test_df["num_punctuations"] =test_df['text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
train_df["num_words_upper"] = train_df["text"].apply(lambda x: len([w for w in str(x ).split() if w.isupper() ]))
test_df["num_words_upper"] = test_df["text"].apply(lambda x: len([w for w in str(x ).split() if w.isupper() ]))
train_df["num_words_title"] = train_df["text"].apply(lambda x: len([w for w in str(x ).split() if w.istitle() ]))
test_df["num_words_title"] = test_df["text"].apply(lambda x: len([w for w in str(x ).split() if w.istitle() ]))
train_df["mean_word_len"] = train_df["text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
test_df["mean_word_len"] = test_df["text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))<feature_engineering>
X_train, X_valid, y_train, y_valid = train_test_split(encoded_categorical_df,y,test_size=0.25 )
author_mapping_dict = {'EAP':0, 'HPL':1, 'MWS':2}
train_y = train_df['author'].map(author_mapping_dict)
 train_id = train_df['id'].values
test_id = test_df['id'].values
train_df["num_words"] = train_df["text"].apply(lambda x: len(str(x ).split()))
test_df["num_words"] = test_df["text"].apply(lambda x: len(str(x ).split()))
train_df["mean_word_len"] = train_df["text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
test_df["mean_word_len"] = test_df["text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
cols_to_drop = ['id', 'text']
train_X = train_df.drop(cols_to_drop+['author'], axis=1)
 test_X = test_df.drop(cols_to_drop, axis=1 )<train_model>
cbr = CatBoostRegressor(logging_level='Silent')
 cbr.fit(X_train,y_train)
 y_pred = cbr.predict(X_valid)
 print("Mean Squared Error",mean_squared_error(y_valid, y_pred))
print("Mean Absolute Error",mean_absolute_error(y_valid, y_pred))
print("R2 Score",r2_score(y_valid, y_pred))
def runXGB(train_X, train_y, test_X, test_y=None, test_X2=None, seed_val=0, child=1, colsample=0.3):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.1
param['max_depth'] = 3
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = child
param['subsample'] = 0.8
param['colsample_bytree'] = colsample
param['seed'] = seed_val
num_rounds = 2000
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=50, verbose_eval=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest, ntree_limit = model.best_ntree_limit)
 if test_X2 is not None:
xgtest2 = xgb.DMatrix(test_X2)
 pred_test_y2 = model.predict(xgtest2, ntree_limit = model.best_ntree_limit)
 return pred_test_y, pred_test_y2, model<compute_train_metric>
<feature_engineering><EOS> 
Id = sample_submission["Id"]
predicted_test = []
for x in cbr.predict(testing_encoding_categorical):
predicted_test.append(x)
 predicted_test_value = pd.DataFrame({ 'Id': Id,
'SalePrice': predicted_test })
 predicted_test_value.to_csv("PredictedTestScore.csv", index=False )