Kaggle competition: Sentiment Analysis on Movie Reviews
featurelist = data.drop('logSalePrice', axis=1 ).columns
pipe = Pipeline([('scaler', RobustScaler()), 
('model', Ridge(fit_intercept=True, random_state=42)) ])
 param_space = {'model__alpha': [1, 5, 10]}
grid = GridSearchCV(pipe, param_grid=param_space, cv=10,
scoring='neg_mean_squared_error')
 grid.fit(data[featurelist], data.logSalePrice )<find_best_params>
import matplotlib.pyplot as plt
import tensorflow as tf
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
grid.best_params_<create_dataframe>
train_data = pd.read_csv('.. /input/sentiment-analysis-on-movie-reviews/train.tsv.zip', sep = '\t')
 train_data.head()
pred = pd.DataFrame({'SalePrice': 10**grid.predict(target[featurelist])}, index=target.Id )<save_to_csv>
test_data = pd.read_csv('.. /input/sentiment-analysis-on-movie-reviews/test.tsv.zip',sep = '\t')
 test_data.head()
pred.to_csv('prediction.csv', index='Id' )<load_from_csv>
print(train_data['Sentiment'].unique())
 train_data['Sentiment'].nunique()
print('reading input files.. ')
 data = pd.read_csv('.. /input/train.csv')
 sampl = pd.read_csv('.. /input/gender_submission.csv' )<load_from_csv>
train_data['Sentiment'].value_counts()
test = pd.read_csv('.. /input/test.csv' )<concatenate>
from tqdm import tqdm
import nltk
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
import re
df = data.append(test, sort = False )<sort_values>
def clean_sentences(df):
reviews = []
for sent in tqdm(df['Phrase']):
text = re.sub("[^a-zA-Z]"," ",sent)
 words = word_tokenize(text.lower())
 new_words = [ ele for ele in words if ele.lower() not in stopwords.words('english')]
lem = WordNetLemmatizer()
lem_words = [lem.lemmatize(i)for i in new_words]
reviews.append(lem_words)
 return(reviews )
totalt = df.isnull().sum().sort_values(ascending=False)
 percent =(df.isnull().sum() /df.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([totalt, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(6 )<merge>
%%time
train_sentences = clean_sentences(train_data)
 test_sentences = clean_sentences(test_data)
 print(len(train_sentences))
print(len(test_sentences))
ticketNum = pd.DataFrame(df.Ticket.value_counts())
 ticketNum.rename(columns = {'Ticket' : 'TicketNum'}, inplace = True)
 ticketNum['TicketId'] = pd.Categorical(ticketNum.index ).codes
ticketNum.loc[ticketNum.TicketNum < 3, 'TicketId'] = -1
df = pd.merge(left = df, right = ticketNum, left_on = 'Ticket',
right_index = True, how = 'left', sort = False)
 df = df.drop(['TicketNum'],axis=1)
 df.head()<feature_engineering>
print(train_data['Phrase'][0])
 print(( " " ).join(train_sentences[0]))
df['FamilyName'] = df.Name.apply(lambda x : str.split(x, ',')[0] )<feature_engineering>
y_target = to_categorical(train_data['Sentiment'].values )
df['FamilySurv'] = 0.5
for _, grup in df.groupby(['FamilyName','Fare']):
if len(grup)!= 1:
for index, row in grup.iterrows() :
smax = grup.drop(index ).Survived.max()
smin = grup.drop(index ).Survived.min()
pid = row.PassengerId
if smax == 1:
df.loc[df.PassengerId == pid, 'FamilySurv'] = 1.0
elif smin == 0:
df.loc[df.PassengerId == pid, 'FamilySurv'] = 0.0
for _, grup in df.groupby(['Ticket']):
if len(grup)!= 1:
for index, row in grup.iterrows() :
if(row.FamilySurv == 0.0 or row.FamilySurv == 0.5):
smax = grup.drop(index ).Survived.max()
smin = grup.drop(index ).Survived.min()
pid = row.PassengerId
if smax == 1:
df.loc[df.PassengerId == pid, 'FamilySurv'] = 1.0
elif smin == 0:
df.loc[df.PassengerId == pid, 'FamilySurv'] = 0.0
df.FamilySurv.value_counts()<feature_engineering>
X_train,X_val,y_train,y_val = train_test_split(train_sentences,y_target,test_size = 0.2,stratify = y_target )
def CabinNum(data):
data.Cabin = data.Cabin.fillna('0')
 regex = re.compile('\s*(\w+)\s*')
 data['CabinNum'] = data.Cabin.apply(lambda x : len(regex.findall(x)))
 CabinNum(df )<count_values>
unique_words = set()
len_max = -1
for sent in tqdm(X_train):
unique_words.update(sent)
 if(len_max < len(sent)) :
len_max = len(sent)
 print('Words in vocab : ' , len(list(unique_words)))
 print('Max_length : ' , len_max )
df.CabinNum.value_counts()<filter>
vocab_size = len(list(unique_words))
embedding_dim = 300
max_length = len_max
trunc_type = 'post'
padding_type = 'post'
oov_tok = '<OOV>'
df.loc[df['Fare'].isnull() ]<filter>
%%time
tokenizer = Tokenizer(num_words = vocab_size,
',
oov_token = oov_tok,
char_level = False)
 tokenizer.fit_on_texts(list(X_train))
X_train = tokenizer.texts_to_sequences(X_train)
 X_train = pad_sequences(X_train,
maxlen = max_length,
padding = padding_type,
truncating = trunc_type)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_val = pad_sequences(X_val,
maxlen = max_length,
padding = padding_type,
truncating = trunc_type)
 X_test = tokenizer.texts_to_sequences(test_sentences)
 X_test = pad_sequences(X_test,
maxlen = max_length,
padding = padding_type,
truncating = trunc_type )
df.loc[(df['Age'] >= 60)&(df['Pclass'] ==3)&(df['Sex'] == 'male')&(df['Embarked'] =='S')]<feature_engineering>
from keras.models import Sequential
from keras.layers import Dense,Bidirectional,LSTM,Activation,Conv1D,MaxPool1D,Dropout
from keras.layers.embeddings import Embedding
df.loc[df['Fare'].isnull() , 'Fare'] = 7<feature_engineering>
model = Sequential()
model.add(Embedding(vocab_size,embedding_dim,input_length = max_length))
model.add(Bidirectional(LSTM(128,dropout = 0.2, recurrent_dropout = 0.2, return_sequences=True)))
 model.add(Bidirectional(LSTM(64, dropout = 0.2, recurrent_dropout = 0.2, return_sequences=False)))
 model.add(Dense(128,activation = 'relu'))
model.add(Dense(y_target.shape[1],activation = 'softmax'))
model.compile(loss = 'categorical_crossentropy',
optimizer = 'adam',
metrics = ['accuracy'])
 model.summary()
def FareFunc(data):
data['FareCat'] = 0
data.loc[data['Fare'] < 8, 'FareCat'] = 0
data.loc[(data['Fare'] >= 8)&(data['Fare'] < 16),'FareCat' ] = 1
data.loc[(data['Fare'] >= 16)&(data['Fare'] < 30),'FareCat' ] = 2
data.loc[(data['Fare'] >= 30)&(data['Fare'] < 45),'FareCat' ] = 3
data.loc[(data['Fare'] >= 45)&(data['Fare'] < 80),'FareCat' ] = 4
data.loc[(data['Fare'] >= 80)&(data['Fare'] < 160),'FareCat' ] = 5
data.loc[(data['Fare'] >= 160)&(data['Fare'] < 270),'FareCat' ] = 6
data.loc[(data['Fare'] >= 270), 'FareCat'] = 7
FareFunc(df )<categorify>
early_stopping = EarlyStopping(min_delta = 0.001,
mode = 'max',
monitor = 'val_acc',
patience = 2)
 callback = [early_stopping]
def FamlSize(data):
data['FamlSize'] = 0
data['FamlSize'] = data['SibSp'] + data['Parch'] + 1
FamlSize(df )<categorify>
%%time
num_epochs = 4
history = model.fit(X_train,y_train,
validation_data =(X_val, y_val),
epochs = num_epochs,
batch_size = 256,
verbose = 1,
callbacks = callback )
def LablFunc(data):
lsr = {'Title','Cabin'}
for i in lsr:
le.fit(data[i].astype(str))
data[i] = le.transform(data[i].astype(str))
LablFunc(df )<predict_on_test>
test_id = test_data['PhraseId']
features = ['Pclass','SibSp','Parch','TicketId','Fare','CabinNum','Title']
def AgeFunc(df):
Etr = ETRg(n_estimators = 200, random_state = 2)
 AgeX_Train = df[features][df.Age.notnull() ]
AgeY_Train = df['Age'][df.Age.notnull() ]
AgeX_Test = df[features][df.Age.isnull() ]
Etr.fit(AgeX_Train,np.ravel(AgeY_Train))
AgePred = Etr.predict(AgeX_Test)
 df.loc[df.Age.isnull() , 'Age'] = AgePred
AgeFunc(df )<feature_engineering>
y_pred = np.argmax(model.predict(X_test), axis = -1 )
def AgeCat(data):
data['AgeCat'] = 0
data.loc[(data['Age'] <= 5), 'AgeCat'] = 0
data.loc[(data['Age'] <= 12)&(data['Age'] > 5), 'AgeCat'] = 1
data.loc[(data['Age'] <= 18)&(data['Age'] > 12), 'AgeCat'] = 2
data.loc[(data['Age'] <= 22)&(data['Age'] > 18), 'AgeCat'] = 3
data.loc[(data['Age'] <= 32)&(data['Age'] > 22), 'AgeCat'] = 4
data.loc[(data['Age'] <= 45)&(data['Age'] > 32), 'AgeCat'] = 5
data.loc[(data['Age'] <= 60)&(data['Age'] > 45), 'AgeCat'] = 6
data.loc[(data['Age'] <= 70)&(data['Age'] > 60), 'AgeCat'] = 7
data.loc[(data['Age'] > 70), 'AgeCat'] = 8
AgeCat(df )<filter>
submission_df = pd.DataFrame({'PhraseId': test_id, 'Sentiment': y_pred})
 submission_df.to_csv('submission_.csv', index=False)
 submission_df.head()