Kaggle competition: Sentiment Analysis on Movie Reviews
train_age_mean = full_data.Age.mean()
train_age_std = full_data.Age.std()
train_age_null = training_data.Age.isnull().sum()
rand_tr_age = np.random.randint(train_age_mean - train_age_std, train_age_mean + train_age_std, size=train_age_null)
 training_data['Age'][np.isnan(training_data['Age'])] = rand_tr_age
training_data['Age'] = training_data['Age'].astype(int)+ 1
test_age_mean = full_data.Age.mean()
test_age_std = full_data.Age.std()
test_age_null = test_data.Age.isnull().sum()
rand_ts_age = np.random.randint(test_age_mean - test_age_std, test_age_mean + test_age_std, size=test_age_null)
 test_data['Age'][np.isnan(test_data['Age'])] = rand_ts_age
test_data['Age'] = test_data['Age'].astype(int)
 training_data.Age =(training_data.Age/15 ).astype(np.int64)
 test_data.Age =(test_data.Age/15 ).astype(np.int64)+ 1<groupby>
lemmatizer = WordNetLemmatizer()
print(training_data[['Age', 'Survived']].groupby(['Age'], as_index = False ).mean() )<define_variables>
set_random_seed(123)
 random.seed(123)
<groupby>
train= pd.read_csv(".. /input/train.tsv", sep="\t")
 test = pd.read_csv(".. /input/test.tsv", sep="\t")
 train.head()
print(training_data[['FamilySize', 'Survived']].groupby(training_data['FamilySize'], as_index=False ).mean() )<feature_engineering>
def clean_sentences(df):
reviews = []
for sent in tqdm(df['Phrase']):
review_text = BeautifulSoup(sent ).get_text()
review_text = re.sub("[^a-zA-Z]"," ", review_text)
 words = word_tokenize(review_text.lower())
 lemma_words = [lemmatizer.lemmatize(i)for i in words]
reviews.append(lemma_words)
 return(reviews)
training_data['isAlone'] = training_data['FamilySize'].map(lambda x: 1 if x == 1 else 0)
 test_data['isAlone'] = test_data['FamilySize'].map(lambda x: 1 if x == 1 else 0 )<drop_column>
train_sentences = clean_sentences(train)
 test_sentences = clean_sentences(test)
 print(len(train_sentences))
print(len(test_sentences))
training_data.drop(labels=['SibSp', 'Parch'], axis=1, inplace=True)
 test_data.drop(labels=['SibSp', 'Parch'], axis=1, inplace=True)
 training_data.head()<feature_engineering>
target=train.Sentiment.values
y_target=to_categorical(target)
 num_classes=y_target.shape[1]
training_data['Ticket_Len'] = training_data['Ticket'].apply(lambda x: len(x))
test_data['Ticket_Len'] = test_data['Ticket'].apply(lambda x: len(x))<groupby>
X_train,X_val,y_train,y_val=train_test_split(train_sentences,y_target,test_size=0.2,stratify=y_target )
print(training_data[['Ticket_Len', 'Survived']].groupby(training_data['Ticket_Len'], as_index=False ).mean() )<drop_column>
unique_words = set()
len_max = 0
for sent in tqdm(X_train):
unique_words.update(sent)
 if(len_max<len(sent)) :
len_max = len(sent)
 print(len(list(unique_words)))
 print(len_max )
training_data.drop(labels='Ticket', axis=1, inplace=True)
 test_data.drop(labels='Ticket', axis=1, inplace=True)
 training_data.head()<data_type_conversions>
tokenizer = Tokenizer(num_words=len(list(unique_words)))
 tokenizer.fit_on_texts(list(X_train))
X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(test_sentences)
 X_train = sequence.pad_sequences(X_train, maxlen=len_max)
 X_val = sequence.pad_sequences(X_val, maxlen=len_max)
 X_test = sequence.pad_sequences(X_test, maxlen=len_max)
 print(X_train.shape,X_val.shape,X_test.shape )
training_data.Fare =(training_data.Fare /20 ).astype(np.int64)+ 1
test_data.Fare =(test_data.Fare /20 ).astype(np.int64)+ 1<groupby>
early_stopping = EarlyStopping(min_delta = 0.001, mode = 'max', monitor='val_acc', patience = 2)
 callback = [early_stopping]
print(training_data[['Fare','Survived']].groupby(['Fare'], as_index = False ).mean() )<define_variables>
model=Sequential()
model.add(Embedding(len(list(unique_words)) ,300,input_length=len_max))
model.add(LSTM(128,dropout=0.5, recurrent_dropout=0.5,return_sequences=True))
model.add(LSTM(64,dropout=0.5, recurrent_dropout=0.5,return_sequences=False))
model.add(Dense(100,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes,activation='softmax'))
model.compile(loss='categorical_crossentropy',optimizer=Adam(lr=0.005),metrics=['accuracy'])
 model.summary()
<count_missing_values>
history=model.fit(X_train, y_train, validation_data=(X_val, y_val),epochs=6, batch_size=256, verbose=1, callbacks=callback )
<count_missing_values><EOS> 
y_pred=model.predict_classes(X_test)
 sub_file = pd.read_csv('.. /input/sampleSubmission.csv',sep=',')
 sub_file.Sentiment=y_pred
sub_file.to_csv('Submission.csv',index=False )