Kaggle competition: Sentiment Analysis on Movie Reviews
print('
{}\tGradient Boost' 
.format(datetime.datetime.now().strftime('%H:%M')))
 parameters = {
'n_estimators':[8000,],
'learning_rate':[0.01],
'max_depth':[2],
'max_features':['sqrt'],
'min_samples_leaf':[10],
'min_samples_split':[5],
'loss':['huber'],
'random_state' :[42],
}
gbr = GradientBoostingRegressor()
clf = GridSearchCV(gbr, parameters, verbose=0, iid=False)
 clf.fit(X_train, y_train)
 gbr = GradientBoostingRegressor(**clf.best_params_)
 print('
Regressor:
', gbr, '
')
 print('{}\tDone! 
.format(datetime.datetime.now().strftime('%H:%M')) )<train_on_grid>
with zipfile.ZipFile('/kaggle/input/sentiment-analysis-on-movie-reviews/train.tsv.zip','r')as zip_ref:
zip_ref.extractall("./sentiment-analysis-on-movie-reviews/")
 with zipfile.ZipFile('/kaggle/input/sentiment-analysis-on-movie-reviews/test.tsv.zip','r')as zip_ref:
zip_ref.extractall("./sentiment-analysis-on-movie-reviews/" )
print('
{}\tK-nearest neighbor' 
.format(datetime.datetime.now().strftime('%H:%M')))
 parameters = {
'algorithm':['auto'],
'leaf_size':[2],
'weights':['uniform', 'distance'],
'metric':['euclidean', 'manhattan'],
'n_neighbors':[10],
}
knn = KNeighborsRegressor()
clf = GridSearchCV(knn, parameters, verbose=0, iid=False)
 clf.fit(X_train, y_train)
 knn = KNeighborsRegressor(**clf.best_params_)
 print('
Regressor:
', knn, '
')
 print('{}\tDone! 
.format(datetime.datetime.now().strftime('%H:%M')) )<train_on_grid>
data_source=pd.read_table("/kaggle/working/sentiment-analysis-on-movie-reviews/train.tsv",sep='\t')
 data_source=data_source[['Phrase','Sentiment']].copy()
data_source
print('
{}\tRandom forest' 
.format(datetime.datetime.now().strftime('%H:%M')))
 parameters = {
'bootstrap': [True],
'max_features': ['auto'],
'min_samples_leaf': [2],
'min_samples_split': [4],
'n_estimators': [1500],
'n_jobs':[-1],
'oob_score':[True]
}
r_forest = RandomForestRegressor()
clf = GridSearchCV(r_forest, parameters, verbose=0, iid=False)
 clf.fit(X_train, y_train)
 knn = RandomForestRegressor(**clf.best_params_)
 print('
Regressor:
', knn, '
')
 print('{}\tDone! 
.format(datetime.datetime.now().strftime('%H:%M')) )<choose_model_class>
dff=[len(i.split(" ")) for i in data_source.Phrase[:10]]
max(dff )
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
<define_search_model>
from transformers import TFBertModel, BertConfig, BertTokenizerFast, TFAutoModel
from tensorflow.keras.layers import Input, Dropout, Dense
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.initializers import TruncatedNormal
from tensorflow.keras.losses import CategoricalCrossentropy
from tensorflow.keras.metrics import CategoricalAccuracy
from tensorflow.keras.utils import to_categorical
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import tensorflow as tf
from sklearn.model_selection import train_test_split
alphas_ridge = [
14.5, 14.6, 14.7, 14.8, 14.9, 15,
15.1, 15.2, 15.3, 15.4, 15.5, 15.6
]
ridge = make_pipeline(
 RobustScaler() ,
RidgeCV(
 alphas=alphas_ridge,
cv=kfolds 
)
)
 print('
Ridge:
',
ridge,
'
' )<define_search_model>
X_train_data, X_validation_data, y_train_data, y_validation_data = train_test_split(data_source.index.values,
data_source.Sentiment.values,
test_size=0.10,
random_state=42,
stratify=data_source.Sentiment)
alphas_lasso = [
5e-05, 0.0001, 0.0002, 0.0003,
0.0004, 0.0005, 0.0006, 0.0007
]
lasso = make_pipeline(
 RobustScaler() ,
LassoCV(
 alphas=alphas_lasso,
max_iter=1e7,
random_state=42,
cv=kfolds 
)
)
 print('
Lasso:
',
lasso,
'
' )<choose_model_class>
data_source['data_type'] = ['not_set']*data_source.shape[0]
data_source.loc[X_train_data, 'data_type'] = 'training'
data_source.loc[X_validation_data, 'data_type'] = 'validation'
alphas_ela = [
0.0001, 0.0002, 0.0003,
0.0004, 0.0005, 0.0006, 0.0007
]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
elasticnet = make_pipeline(
 RobustScaler() ,
ElasticNetCV(
 max_iter=1e7,
alphas=alphas_ela,
cv=kfolds,
l1_ratio=e_l1ratio 
)
)
 print('
Elastic:
',
elasticnet,
'
' )<train_on_grid>
data_source.isnull().sum()
%%time
parameters = {
'C': [120],
'gamma': [0.0003]
}
svr_ = SVR()
clf = GridSearchCV(svr_, parameters, verbose=0, iid=False)
 clf.fit(X_train, y_train)
 svr = make_pipeline(
 RobustScaler() ,
SVR(**clf.best_params_)
)
 print('
SVR:
',
svr,
'
' )<choose_model_class>
data_source[data_source.data_type=='training'].Phrase
krr = KernelRidge()<choose_model_class>
max_token_length = max(dff)+3
number_of_samples = len(data_source)
 bert = 'bert-base-cased'
config = BertConfig.from_pretrained(bert)
 config.output_hidden_states = False
tokenizer = BertTokenizerFast.from_pretrained(pretrained_model_name_or_path = bert, config = config)
print('Stacking: \t stack_gen')
 stack_gen = StackingCVRegressor(
 regressors=(ridge, lasso, elasticnet, lightgbm, XGBoost, gbr),
meta_regressor=XGBoost,
use_features_in_secondary=True 
)
 print('Done!
')
 print('Stacking: \t stack_gen2')
 stack_gen2 = StackingCVRegressor(
 regressors=(ridge, lasso, elasticnet, lightgbm, XGBoost, gbr),
meta_regressor=lightgbm,
use_features_in_secondary=True 
)
 print('Done!
')
 print('Stacking: \t stack_gen3')
 stack_gen3 = StackingCVRegressor(
 regressors=(ridge, lasso, elasticnet, lightgbm, XGBoost, gbr),
meta_regressor=ridge,
use_features_in_secondary=True 
)
 print('Done!' )<train_model>
def map_function(input_ids, masks,labels):
return {'input_ids': input_ids, 'attention_mask': masks},labels
models = {
'svr':svr,
'KernelRidge':krr,
'Ridge':ridge,
'Lightgbm':lightgbm,
'XGBoost':XGBoost,
'GradientBoost':gbr,
'Lasso':lasso,
'Elasticnet':elasticnet,
'Random forest':r_forest,
'KNN':knn
}
def calc(model):
return int(np.sqrt(metrics.mean_squared_error(
 np.expm1(y_test), np.expm1(model.predict(X_test)))))
 liste = []
for one, two in models.items() :
print('Fitting:\t{}'.format(one))
two.fit(X_train, y_train)
 print('Done! Error:\t{}
'.format(calc(two)))
 df = pd.DataFrame(
 data=[calc(two)],
index=[one],
columns=['error'] 
)
 liste.append(df )<concatenate>
x_senti = to_categorical(data_source[data_source.data_type=='training'].Sentiment)
 x = tokenizer(
 text=data_source[data_source.data_type=='training'].Phrase.to_list() ,
add_special_tokens=True,
max_length=max_token_length,
truncation=True,
padding=True,
return_tensors='tf',
return_token_type_ids = False,
return_attention_mask = True,
verbose = True)
 train_dataset=tf.data.Dataset.from_tensor_slices(( x['input_ids'], x['attention_mask'], x_senti))
resList = pd.concat(liste, axis=0 )<compute_test_metric>
train_dataset = train_dataset.map(map_function )
print('stack_gen: \t fitting...')
 stack_gen_model = stack_gen.fit(np.array(X_test), np.array(y_test))
sg_error = int(np.sqrt(metrics.mean_squared_error(np.expm1(y_test), np.expm1(stack_gen_model.predict(np.array(X_test))))))
print('stack_gen: \t done!')
 print('error:\t\t', sg_error, '
')
 print('stack_gen2: \t fitting...')
 stack_gen_model2 = stack_gen2.fit(np.array(X_test), np.array(y_test))
sg_error2 = int(np.sqrt(metrics.mean_squared_error(np.expm1(y_test), np.expm1(stack_gen_model2.predict(np.array(X_test))))))
print('stack_gen2: \t done!')
 print('error:\t\t', sg_error2, '
')
 print('stack_gen3: \t fitting...')
 stack_gen_model3 = stack_gen3.fit(np.array(X_test), np.array(y_test))
sg_error3 = int(np.sqrt(metrics.mean_squared_error(np.expm1(y_test), np.expm1(stack_gen_model3.predict(np.array(X_test))))))
print('stack_gen3: \t done!')
 print('error:\t\t', sg_error3)
 sg = pd.DataFrame(index=['stack_gen'], data=sg_error, columns=['error'])
 sg2 = pd.DataFrame(index=['stack_gen2'], data=sg_error2, columns=['error'])
 sg3 = pd.DataFrame(index=['stack_gen3'], data=sg_error3, columns=['error'] )<concatenate>
batch_size = 32
train_dataset = train_dataset.shuffle(1000 ).batch(batch_size, drop_remainder=True )
resList = resList.append(sg)
 resList = resList.append(sg2)
 resList = resList.append(sg3 )<predict_on_test>
y_senti = to_categorical(data_source[data_source.data_type=='validation'].Sentiment)
 v= tokenizer(
 text=data_source[data_source.data_type=='validation'].Phrase.to_list() ,
add_special_tokens=True,
max_length=max_token_length,
truncation=True,
padding=True,
return_tensors='tf',
return_token_type_ids = False,
return_attention_mask = True,
verbose = True)
 validation_dataset = tf.data.Dataset.from_tensor_slices(( v['input_ids'], v['attention_mask'], y_senti))
validation_dataset = validation_dataset.map(map_function)
 validation_dataset = validation_dataset.shuffle(1000 ).batch(batch_size, drop_remainder=True )
def blend(data):
return(
(0.08 * ridge.predict(data)) + 
(0.11 * lightgbm.predict(data)) + 
(0.13 * XGBoost.predict(data)) + 
(0.10 * gbr.predict(data)) + 
(0.15 * stack_gen_model.predict(np.array(data)))+ 
(0.16 * stack_gen_model2.predict(np.array(data)))+ 
(0.16 * stack_gen_model3.predict(np.array(data)))+ 
(0.06 * lasso.predict(data)) + 
(0.05 * elasticnet.predict(data)) 
)
 blended_predictions = blend(test )<compute_test_metric>
input_ids = tf.keras.Input(shape=(max_token_length,), name='input_ids', dtype='int32')
 attention_mask = tf.keras.Input(shape=(max_token_length,), name='attention_mask', dtype='int32')
 inputs = {'input_ids': input_ids, 'attention_mask': attention_mask}
bert = TFAutoModel.from_pretrained('bert-base-cased')
 embeddings = bert.bert(inputs)[1]
xis = tf.keras.layers.Dense(1024,activation='relu' )(embeddings)
 yhat = tf.keras.layers.Dense(sent_values_array.max() +1, activation='softmax', name='outputs' )(xis)
x = blend(X_test)
 bl = pd.DataFrame(
 index=['blend'],
data=int(np.sqrt(metrics.mean_squared_error(np.expm1(y_test), np.expm1(x)))) ,
columns=['error'] 
)
 resList = resList.append(bl )<save_to_csv>
optimizer = tf.keras.optimizers.Adam(lr=1e-5, decay=1e-6)
 loss = tf.keras.losses.CategoricalCrossentropy()
accuracy = tf.keras.metrics.CategoricalAccuracy('accuracy' )
res = pd.DataFrame(data=np.expm1(blend(test)) , index=ID.Id, columns=['SalePrice'])
 q1 = res['SalePrice'].quantile(0.005)
 q2 = res['SalePrice'].quantile(0.995)
 res['SalePrice'] = res['SalePrice'].apply(lambda x: x if x > q1 else x*0.78)
 res['SalePrice'] = res['SalePrice'].apply(lambda x: x if x < q2 else x*1.10)
 res.to_csv('1_submission.csv' )<load_from_csv>
model.compile(optimizer=optimizer, loss=loss, metrics=[accuracy] )
dataset = pd.read_csv('.. /input/train.csv' )<prepare_x_and_y>
history1 = model.fit(
 train_dataset,
validation_data= validation_dataset,
epochs=8 )
features= [ 'Pclass','Sex','Age','SibSp','Parch','Fare','Embarked']
x = dataset[features]
y = dataset['Survived']<count_missing_values>
model.save('./sentiment-analysis-on-movie-reviews/Movie_sentiment_analysis_model' )
x.isnull().sum()<feature_engineering>
test=pd.read_table("/kaggle/working/sentiment-analysis-on-movie-reviews/test.tsv",sep='\t' )
x['Age'] = x['Age'].fillna(x['Age'].median())
 x['Embarked']= x['Embarked'].fillna(x['Embarked'].value_counts().index[0] )<count_missing_values>
x_test = tokenizer(
 text=test.Phrase.to_list() ,
add_special_tokens=True,
max_length=max_token_length,
truncation=True,
padding=True,
return_tensors='tf',
return_token_type_ids = False,
return_attention_mask = True,
verbose = True )
x.isnull().sum()<categorify>
test_items=tf.data.Dataset.from_tensor_slices(( x_test['input_ids'],x_test['attention_mask']))
LE = LabelEncoder()
x['Sex'] = LE.fit_transform(x['Sex'])
 x['Embarked'] = LE.fit_transform(x['Embarked'] )<count_missing_values>
def map_func(input_ids, masks):
return {'input_ids': input_ids, 'attention_mask': masks}
test_items = test_items.map(map_func)
 test_items = test_items.batch(32 )
y.isnull().sum()<split>
predictions=model.predict(test_items ).argmax(axis=-1 )
x_train,x_test,y_train,y_test = train_test_split(x,y,test_size = 0.1,random_state =0 )<train_model>
submission = pd.DataFrame()
submission['PhraseId'] = test['PhraseId']
submission['Sentiment'] = predictions
submission.to_csv("submission.csv", index=False)
 submission.head()