Kaggle competition: Data Mining Lab2
<SOS> metric: meanfscore Kaggle data source: data-mining-lab2<feature_engineering>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import keras
from sklearn import metrics
from sklearn.preprocessing import LabelEncoder,OneHotEncoder
from keras.models import Model
from keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding
from keras.optimizers import RMSprop
from keras.preprocessing.text import Tokenizer
from keras.preprocessing import sequence
from keras.callbacks import EarlyStopping
full['BsmtFinSF1'] = full['BsmtFinSF1'].fillna(0)
 full['BsmtFinSF2'] = full['BsmtFinSF2'].fillna(0)
 full['TotalBsmtSF'] = full['TotalBsmtSF'].fillna(0)
 full['BsmtUnfSF'] = full['BsmtUnfSF'].fillna(0)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )<filter>
import json
from pandas.io.json import json_normalize
full[full['BsmtFullBath'].isnull() ]<data_type_conversions>
raw_data=pd.read_json(".. /input/datamininglab2/tweets_DM.json",lines=True)
 tweets=json_normalize(data=raw_data['_source'])
 identify=pd.read_csv(".. /input/datamininglab2/data_identification.csv")
 emotion=pd.read_csv(".. /input/datamininglab2/emotion.csv" )
full['BsmtFullBath'] = full['BsmtFullBath'].fillna(0)
 full['BsmtHalfBath'] = full['BsmtHalfBath'].fillna(0)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )<filter>
tweets=tweets.rename(index=str,columns={"tweet.text":"text", "tweet.tweet_id":"tweet_id",
"tweet.hashtags":"hashtags"})
 tweets=pd.merge(tweets,identify, on="tweet_id")
 train_df=tweets[tweets["identification"] == "train"]
test_df=tweets[tweets["identification"] == "test"]
train_df=pd.merge(train_df,emotion, on="tweet_id")
 test_df["emotion"]=""
train_df.drop(columns=["identification"],inplace=True)
 test_df.drop(columns=["identification"],inplace=True)
 train_df.set_index("tweet_id",inplace=True)
 test_df.set_index("tweet_id",inplace=True )
full[full['Functional'].isnull() ]<predict_on_test>
train_df.to_pickle("train_df.pkl")
 test_df.to_pickle("test_df.pkl" )
full[full['MasVnrArea'].isnull() ]<sort_values>
train_df = pd.read_pickle(".. /input/dm-competition-tweets-emotion/train_df.pkl")
 test_df = pd.read_pickle(".. /input/dm-competition-tweets-emotion/test_df.pkl" )
full['MasVnrArea'] = full['MasVnrArea'].fillna(0)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )<filter>
tknzr = TweetTokenizer()
full[full['GarageYrBlt'].isnull() ]<feature_engineering>
tknzr = TweetTokenizer(strip_handles=True, reduce_len=True)
 s1 = '@remy: This is waaaaayyyy too much for you!!!!!!'
tknzr.tokenize(s1 )
null_df = full[full['GarageYrBlt'].isnull() ]
null_df['GarageAge'] = null_df['YearBuilt'].apply(lambda x: 2019 - x)
 null_df['GarageYrBlt'] = null_df['YearBuilt']
full[full['GarageYrBlt'].isnull() ] = null_df
missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )<filter>
tknzr = TweetTokenizer(preserve_case=False)
 tfidf = TfidfVectorizer(max_features=20000, stop_words='english',
tokenizer=tknzr.tokenize)
 tfidf.fit(train_df['text'] )
full[full['LotFrontage'].isnull() ]<predict_on_test>
X_train = tfidf.transform(train_df['text'])
 X_train.shape
full['Condition1'] = full['Condition1'].fillna(0)
 full['Condition2'] = full['Condition2'].fillna(0)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )<sort_values>
X_test = tfidf.transform(test_df['text'])
 X_test.shape
full['Alley'] = full['Alley'].fillna(-1)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )<feature_engineering>
y_train = train_df['emotion']
y_test = test_df['emotion']
full['LogLotArea'] = np.log(full['LotArea'])
 full['LogLotFrontage'] = np.log(full['LotFrontage'] )<predict_on_test>
model_compare=pd.read_csv(".. /input/dm-competition-tweets-emotion/final.csv")
 model_compare
missing_mask = full.isnull() ['LotFrontage']
filled_mask = full.notnull() ['LotFrontage']
predict = full[missing_mask]
train = full[filled_mask]
X_train = train['LogLotArea']
y_train = train['LogLotFrontage']
X_predict = predict['LogLotArea']
model = XGBRegressor(n_jobs = -1)
 model.fit(np.array(X_train ).reshape(-1,1), np.array(y_train))
y_predict = model.predict(np.array(X_predict ).reshape(-1,1))
y_predict.shape
full['LogLotFrontage'][missing_mask] = y_predict<sort_values>
lr = LogisticRegression(C=6,n_jobs=-1,max_iter=1000)
 lr.fit(X_train,y_train )
missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )<feature_engineering>
pred_result_lr = lr.predict(X_test)
 pred_result_lr.shape
full['LotFrontage'][missing_mask] = np.exp(full['LogLotFrontage'][missing_mask])
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )<sort_values>
test_df['emotion']=pred_result_lr
test_df.drop(columns=['hashtags','text'],inplace=True)
 test_df.index.rename('id',inplace=True)
 test_df.columns=['emotion']
test_df.to_csv('lr_tfidf.csv' )
full['Functional'] = full['Functional'].fillna(0)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )<filter>
train_df = pd.read_pickle(".. /input/dm-competition-tweets-emotion/train_df.pkl")
 test_df = pd.read_pickle(".. /input/dm-competition-tweets-emotion/test_df.pkl" )
full[full['GarageCars'].isnull() ]['GarageCond']<sort_values>
max_words = 20000
max_len = 300
tok = Tokenizer(num_words=max_words)
 tok.fit_on_texts(train_df['text'] )
full['GarageCars'] = full['GarageCars'].fillna(-1)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )<filter>
train_seq = tok.texts_to_sequences(train_df['text'])
 test_seq = tok.texts_to_sequences(test_df['text'])
 train_seq_mat = sequence.pad_sequences(train_seq,maxlen=max_len)
 test_seq_mat = sequence.pad_sequences(test_seq,maxlen=max_len)
 print(train_seq_mat.shape)
 print(test_seq_mat.shape )
full[full['GarageArea'].isnull() ]['GarageCond']<sort_values>
label_encoder = LabelEncoder()
label_encoder.fit(y_train)
 print('check label: ', label_encoder.classes_)
 print('
print('y_train[0:4]:
', y_train[0:4])
 print('
y_train.shape: ', y_train.shape)
 print('y_test.shape: ', y_test.shape)
 def label_encode(le, labels):
enc = le.transform(labels)
 return keras.utils.to_categorical(enc)
 def label_decode(le, one_hot_label):
dec = np.argmax(one_hot_label, axis=1)
 return le.inverse_transform(dec)
 y_train = label_encode(label_encoder, y_train)
 y_test = label_encode(label_encoder, y_test)
 print('
print('y_train[0:4]:
', y_train[0:4])
 print('
y_train.shape: ', y_train.shape)
 print('y_test.shape: ', y_test.shape )
full['GarageArea'] = full['GarageArea'].fillna(-1)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )<normalization>
input_shape = X_train.shape[1]
print('input_shape: ', input_shape)
 output_shape = len(label_encoder.classes_)
 print('output_shape: ', output_shape )
skewed_feats = full.apply(lambda x: st.skew(x.dropna())).sort_values(ascending=False)
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
walk_through_lambdas = True
def getBestLambda(data):
statistic = []
pvalue = []
bclambda_range = np.linspace(-3, 3, 1000)
 for bclambda in bclambda_range:
transform = boxcox1p(data, bclambda)
 stat, pval = normaltest(transform)
 statistic.append(stat)
 pvalue.append(pval)
 lowest_stat = 8000
closest = -10
best_stat = -1
best_pval = -1
for i in range(len(bclambda_range)) :
if statistic[i] < lowest_stat:
lowest_stat = statistic[i]
best_stat = i
if(abs(pvalue[i])- 1)> closest:
closest =(abs(pvalue[i])- 1)
 best_pval = i
return bclambda_range[best_pval], bclambda_range[best_stat]
i = 0
with progressbar.ProgressBar(max_value=len(skewed_features)) as bar:
for feat in skewed_features:
if walk_through_lambdas:
full["BC_{}".format(feat)] = boxcox1p(full[feat], getBestLambda(full[feat])[0])
 else:
full["BC_{}".format(feat)] = boxcox1p(full[feat],.15)
 i+=1
bar.update(i)
 missing_values = full.isnull().sum()
for column in missing_values[missing_values>0].keys() :
print("Dropping feature {}".format(column))
full.drop(column, axis = 1, inplace = True )<prepare_x_and_y>
inputs = Input(name='inputs',shape=[max_len])
 layer = Embedding(max_words+1,128,input_length=max_len )(inputs)
 layer = LSTM(128 )(layer)
 layer = Dense(128,activation="relu",name="FC1" )(layer)
 layer = Dropout(0.5 )(layer)
 layer = Dense(output_shape,activation="softmax",name="FC2" )(layer)
 model = Model(inputs=inputs,outputs=layer)
 model.summary()
model.compile(loss="categorical_crossentropy",optimizer=RMSprop() ,metrics=["accuracy"] )
X_train = full[full['Id'].isin(df_train['Id'])]
X_train.drop('Id', axis = 1, inplace = True)
 X_test = full[full['Id'].isin(df_test['Id'])]
X_test.drop('Id', axis = 1, inplace = True)
 y_train = df_train['SalePrice']
print("X_train shape is {}, y_train shape is {} and X_test shape is {}".format(
 X_train.shape,
y_train.shape,
X_test.shape 
))<normalization>
model_fit = model.fit(train_seq_mat,y_train,batch_size=128,epochs=3,
callbacks=[EarlyStopping(monitor='val_loss',min_delta=0.0001)])
scaler = StandardScaler()
X_train_scaled = scaler.fit(X_train ).transform(X_train)
 X_test_scaled = scaler.transform(X_test)
 bclambda = getBestLambda(y_train)[0]
y_train_transformed = boxcox1p(y_train, bclambda )<import_modules>
pred_result_lstm = label_decode(label_encoder, model.predict(test_seq_mat, batch_size=128))
pred_result_lstm[:5]
n_folds = 5
def rmse_cv(model, X, y, cv=5):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring = "neg_mean_squared_error", cv = cv))
return rmse
def rmse(y_actual, y_predicted):
return sqrt(mean_squared_error(y_actual, y_predicted)) 
<normalization>
test_df['emotion']=pred_result_lstm
test_df.drop(columns=['hashtags','text'],inplace=True)
 test_df.index.rename('id',inplace=True)
 test_df.columns=['emotion']
test_df.to_csv('keras_tfidf.csv' )
<features_selection><EOS> 
model_compare=pd.read_csv(".. /input/dm-competition-tweets-emotion/final.csv")
 model_compare