Kaggle competition: Bag of Words Meets Bags of Popcorn
warnings.filterwarnings("ignore")
<compute_test_metric>
train = pd.read_csv('.. /input/labeledTrainData.tsv',delimiter = '\t')
 test = pd.read_csv('.. /input/testData.tsv',delimiter = '\t')
 train.shape, test.shape
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )<choose_model_class>
print("number of rows for sentiment 1: {}".format(len(train[train.sentiment == 1])))
 print("number of rows for sentiment 0: {}".format(len(train[train.sentiment == 0])) )
def create_model(data, catcols):
inputs = []
outputs = []
for c in catcols:
num_unique_values = int(data[c].nunique())
 embed_dim = int(min(np.ceil(( num_unique_values)/2), 50))
inp = layers.Input(shape=(1,))
out = layers.Embedding(num_unique_values + 1, embed_dim, name=c )(inp)
 out = layers.SpatialDropout1D(0.3 )(out)
 out = layers.Reshape(target_shape=(embed_dim,))(out)
 inputs.append(inp)
 outputs.append(out)
 x = layers.Concatenate()(outputs)
 x = layers.BatchNormalization()(x)
 x = layers.Dense(300, activation="relu" )(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.BatchNormalization()(x)
 x = layers.Dense(300, activation="relu" )(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.BatchNormalization()(x)
 y = layers.Dense(2, activation="softmax" )(x)
 model = Model(inputs=inputs, outputs=y)
 return model<load_from_csv>
train['length'] = train['review'].apply(len)
 train.head()
train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv")
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv")
 sample = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv" )<concatenate>
train[train['length'] == 13708]['review'].iloc[0]
test["target"] = -1
data = pd.concat([train, test] ).reset_index(drop=True )<define_variables>
def clean_text(raw_text):
raw_text = BeautifulSoup(raw_text ).get_text()
letters_only = re.sub("[^a-zA-Z]", " ", raw_text)
 words = letters_only.lower().split()
stops = set(stopwords.words("english"))
return [w for w in words if not w in stops]
features = [x for x in train.columns if x not in ["id", "target"]]<categorify>
train['clean_review'] = train['review'].apply(clean_text)
 train['length_clean_review'] = train['clean_review'].apply(len)
 train.head()
for feat in features:
lbl_enc = preprocessing.LabelEncoder()
data[feat] = lbl_enc.fit_transform(data[feat].values )<filter>
print(train[train['length_clean_review'] == 4]['review'].iloc[0])
 print('------After Cleaning------')
 print(train[train['length_clean_review'] == 4]['clean_review'].iloc[0] )
train = data[data.target != -1].reset_index(drop=True)
 test = data[data.target == -1].reset_index(drop=True)
 test_data = [test.loc[:, features].values[:, k] for k in range(test.loc[:, features].values.shape[1])]<split>
from sklearn.feature_extraction.text import CountVectorizer
oof_preds = np.zeros(( len(train)))
 test_preds = np.zeros(( len(test)))
 skf = StratifiedKFold(n_splits=50)
 for train_index, test_index in skf.split(train, train.target.values):
X_train, X_test = train.iloc[train_index, :], train.iloc[test_index, :]
X_train = X_train.reset_index(drop=True)
 X_test = X_test.reset_index(drop=True)
 y_train, y_test = X_train.target.values, X_test.target.values
model = create_model(data, features)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[auc])
 X_train = [X_train.loc[:, features].values[:, k] for k in range(X_train.loc[:, features].values.shape[1])]
X_test = [X_test.loc[:, features].values[:, k] for k in range(X_test.loc[:, features].values.shape[1])]
es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=5,
verbose=1, mode='max', baseline=None, restore_best_weights=True)
 rlr = callbacks.ReduceLROnPlateau(monitor='val_auc', factor=0.5,
patience=3, min_lr=1e-6, mode='max', verbose=1)
 model.fit(X_train,
utils.to_categorical(y_train),
validation_data=(X_test, utils.to_categorical(y_test)) ,
verbose=1,
batch_size=1024,
callbacks=[es, rlr],
epochs=100 
)
 valid_fold_preds = model.predict(X_test)[:, 1]
test_fold_preds = model.predict(test_data)[:, 1]
oof_preds[test_index] = valid_fold_preds.ravel()
test_preds += test_fold_preds.ravel()
print(metrics.roc_auc_score(y_test, valid_fold_preds))
K.clear_session()<compute_test_metric>
bow_transform = CountVectorizer(analyzer=clean_text ).fit(train['review'])
 print(len(bow_transform.vocabulary_))
print("Overall AUC={}".format(metrics.roc_auc_score(train.target.values, oof_preds)) )<save_to_csv>
review_bow = bow_transform.transform(train['review'] )
test_preds /= 50
test_ids = test.id.values
print("Saving submission file")
 submission = pd.DataFrame.from_dict({
'id': test_ids,
'target': test_preds
})
 submission.to_csv("submission.csv", index=False )<install_modules>
sparsity =(100.0 * review_bow.nnz /(review_bow.shape[0] * review_bow.shape[1]))
print('sparsity: {}'.format(sparsity))
!pip install target_encoding 
<import_modules>
tfidf_transformer = TfidfTransformer().fit(review_bow)
 tfidf1 = tfidf_transformer.transform(bow1)
 print(tfidf1 )
import pandas as pd
from target_encoding import TargetEncoderClassifier, TargetEncoder
from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LinearRegression, LogisticRegression<load_from_csv>
print(tfidf_transformer.idf_[bow_transform.vocabulary_['war']])
 print(tfidf_transformer.idf_[bow_transform.vocabulary_['book']] )
train=pd.read_csv(".. /input/cat-in-the-dat/train.csv")
 test=pd.read_csv(".. /input/cat-in-the-dat/test.csv")
 sample_submission = pd.read_csv('.. /input/cat-in-the-dat/sample_submission.csv')
 cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42 )<categorify>
review_tfidf = tfidf_transformer.transform(review_bow)
 print(review_tfidf.shape )
len_uniques = []
for c in train.columns.drop(['id', 'target']):
le = LabelEncoder()
le.fit(pd.concat([train[c], test[c]]))
train[c] = le.transform(train[c])
 test[c] = le.transform(test[c])
 print(c, len(le.classes_))
len_uniques.append(len(le.classes_))
X = train.drop(['target', 'id'], axis=1)
 y = train['target']<define_variables>
X_train, X_test, y_train, y_test = train_test_split(train['review'], train['sentiment'], test_size=0.22, random_state=101)
 len(X_train), len(X_test), len(X_train)+ len(X_test )
ALPHA = 75
MAX_UNIQUE = max(len_uniques)
 FEATURES_COUNT = X.shape[1]<compute_train_metric>
def pred(predicted,compare):
cm = pd.crosstab(compare,predicted)
 TN = cm.iloc[0,0]
FN = cm.iloc[1,0]
TP = cm.iloc[1,1]
FP = cm.iloc[0,1]
print("CONFUSION MATRIX ------->> ")
 print(cm)
 print()
print('Classification paradox :------->>')
 print('Accuracy :- ', round(((TP+TN)*100)/(TP+TN+FP+FN),2))
print()
print('False Negative Rate :- ',round(( FN*100)/(FN+TP),2))
print()
print('False Postive Rate :- ',round(( FP*100)/(FP+TN),2))
print()
print(classification_report(compare,predicted))
lin = LogisticRegression()
score = cross_val_score(lin, X_train, y, scoring='roc_auc', cv=cv)
 print(f'score: {score.mean() :.4}, std: {score.std() :.4}')
 lin.fit(X_train, y)
 pred_lin = lin.predict_proba(X_test)[:,1]<save_to_csv>
pipeline = Pipeline([ 
('bow', CountVectorizer(analyzer=clean_text)) , 
('tfidf', TfidfTransformer()), 
('classifier', LogisticRegression(random_state=101)) ,
])
 pipeline.fit(X_train,y_train)
 predictions = pipeline.predict(X_train)
 pred(predictions,y_train )
sample_submission['target'] = pred_enc + pred_lin
sample_submission.to_csv('submission.csv', index=False )<load_from_csv>
predictions = pipeline.predict(X_test)
 pred(predictions,y_test )
train = pd.read_csv("/kaggle/input/train.csv")
 test = pd.read_csv("/kaggle/input/test.csv")
 submission = pd.read_csv("/kaggle/input/sampleSubmission.csv" )<concatenate>
pipeline = Pipeline([ 
('bow', CountVectorizer(analyzer=clean_text)) , 
('tfidf', TfidfTransformer()), 
('classifier', MultinomialNB()),
])
 pipeline.fit(X_train,y_train)
 predictions = pipeline.predict(X_train)
 pred(predictions,y_train )
all_data_temp = pd.concat([train, test])
 all_data_temp<concatenate>
predictions = pipeline.predict(X_test)
 pred(predictions,y_test )
all_data = pd.concat([train, test], ignore_index=True)
 all_data<feature_engineering>
pipeline = Pipeline([ 
('bow', CountVectorizer(analyzer=clean_text)) , 
('tfidf', TfidfTransformer()), 
('classifier', RandomForestClassifier(n_estimators = 500)) ,
])
 pipeline.fit(X_train,y_train)
 predictions = pipeline.predict(X_train)
 pred(predictions,y_train )
all_data['date'] = all_data['datetime'].apply(lambda x: x.split() [0])
 all_data['year'] = all_data['datetime'].apply(lambda x: x.split() [0].split('-')[0])
 all_data['month'] = all_data['datetime'].apply(lambda x: x.split() [0].split('-')[1])
 all_data['hour'] = all_data['datetime'].apply(lambda x: x.split() [1].split(':')[0])
 all_data["weekday"] = all_data['date'].apply(lambda dateString : datetime.strptime(dateString,"%Y-%m-%d" ).weekday() )<data_type_conversions>
predictions = pipeline.predict(X_test)
 pred(predictions,y_test )
categorical_features = ['season', 'holiday', 'workingday', 'weather', 'weekday', 'month', 'year', 'hour']
for feature in categorical_features:
all_data[feature] = all_data[feature].astype("category" )<prepare_x_and_y>
pipeline_logit = Pipeline([ 
('bow', CountVectorizer(analyzer=clean_text)) , 
('tfidf', TfidfTransformer()), 
('classifier', LogisticRegression(random_state=101)) ,
])
 pipeline_logit.fit(train['review'],train['sentiment'])
 test['sentiment'] = pipeline_logit.predict(test['review'] )
train = all_data[pd.notnull(all_data['count'])]
test = all_data[~pd.notnull(all_data['count'])]
y = train['count']<drop_column>
output.to_csv("output.csv", index=False, quoting=3 )