Kaggle competition: Classifying Movie Reviews
%%time
transform_data[numerical_features].apply(lambda x: kurt(x.dropna())).sort_values(ascending=False )<feature_engineering>
training_data, training_labels = load_svmlight_file('.. /input/movie-ratings/movie-ratings/data-splits/data.train')
 testing_data, testing_labels = load_svmlight_file('.. /input/movie-ratings/movie-ratings/data-splits/data.test')
 testing_data = csr_matrix(( testing_data.data, testing_data.indices, testing_data.indptr), shape=(testing_data.shape[0], training_data.shape[1]))
%%time
skewed_features = {feature: skew(transform_data[feature].dropna() -transform_data[feature].min())for feature in numerical_features if skew(transform_data[feature].dropna() -transform_data[feature].min())>=.5}
print(skewed_features)
 boxcox1p_skews = {}
log1p_skews = {}
exp_skews = {}
power_skews = {}
best_skews = {}
methods = {}
tmp = pd.DataFrame()
for feature in skewed_features.keys() :
tmp[feature] = boxcox1p(transform_data[feature]-transform_data[feature].min() , boxcox_normmax(transform_data[feature]-transform_data[feature].min() + 1))
boxcox1p_skews[feature] = skew(tmp[feature].dropna())
 tmp[feature] = np.log1p(transform_data[feature]-transform_data[feature].min())
 log1p_skews[feature] = skew(tmp[feature].dropna())
 tmp[feature] = np.exp(transform_data[feature]-transform_data[feature].min())
 exp_skews[feature] = skew(tmp[feature].dropna())
 tmp[feature] =(transform_data[feature]-transform_data[feature].min())**.5
power_skews[feature] = skew(tmp[feature].dropna())
 best_skews[feature] = min(boxcox1p_skews[feature],log1p_skews[feature],exp_skews[feature],power_skews[feature])
 methods[feature] = 'boxcox' if best_skews[feature]==boxcox1p_skews[feature] else('log' if best_skews[feature]==log1p_skews[feature] else('exp' if best_skews[feature]==exp_skews[feature] else('power' if best_skews[feature]==power_skews[feature] else 'original')))
 print(methods)
 df_skew = pd.DataFrame(index=skewed_features.keys() , columns=['Skew', 'Skew after boxcox1p'])
 df_skew['Skew Original'] = skewed_features.values()
df_skew['Skew after boxcox1p'] = boxcox1p_skews.values()
df_skew['Skew after log1p'] = log1p_skews.values()
df_skew['Skew after exp'] = exp_skews.values()
df_skew['Skew after power'] = power_skews.values()
df_skew['Skew after compare'] = best_skews.values()
fig = plt.figure(figsize=(22, 6))
sns.pointplot(x=df_skew.index, y='Skew Original', data=df_skew, markers=['o'], linestyles=['-'], label='original')
 sns.pointplot(x=df_skew.index, y='Skew after boxcox1p', data=df_skew, markers=['x'], linestyles=['--'], color='dodgerblue', label='boxcox1p')
 sns.pointplot(x=df_skew.index, y='Skew after log1p', data=df_skew, markers=['s'], linestyles=['--'], color='peru', label='log1p')
 sns.pointplot(x=df_skew.index, y='Skew after exp', data=df_skew, markers=['*'], linestyles=['--'], color='darkorchid', label='exp')
 sns.pointplot(x=df_skew.index, y='Skew after power', data=df_skew, markers=['+'], linestyles=['--'], color='yellow', label='power0.5')
 sns.pointplot(x=df_skew.index, y='Skew after compare', data=df_skew, markers=['v'], linestyles=['--'], color='lawngreen', label='best')
 plt.xlabel('Skewed Features', size=20, labelpad=12.5)
 plt.ylabel('Skewness', size=20, labelpad=12.5)
 plt.tick_params(axis='x', labelsize=11)
 plt.tick_params(axis='y', labelsize=15)
 plt.xticks(rotation=70)
 plt.title('Skewed Features Before and After Several Transformation Method', size=20)
 plt.legend(loc='best', shadow=True)
 plt.show()<feature_engineering>
class SVM:
def __init__(self, learning_rate=0.1, reg_const=0.05, l1=False, num_iters=10, batch_size=200):
self.W = None
self.learning_rate = learning_rate
self.reg_const = reg_const
self.l1 = l1
if self.l1:
self.reg_fun = lambda x: np.max(np.sum(np.abs(x), axis=0))
self.reg_fun_grad = lambda x: np.sum([np.sign(val)for val in x.flatten() if not np.isclose(val,0, atol=1e-5)])
 else:
self.reg_fun = lambda x: 0.5*np.sum(x*x)
 self.reg_fun_grad=lambda x:x
self.num_iters = num_iters
self.batch_size = batch_size
def loss(self,X_batch, y_batch):
num_train = X_batch.shape[0]
num_classes = self.W.shape[1]
scores = X_batch.dot(self.W)
 correct_class_scores = scores[range(num_train), y_batch].reshape(-1,1)
 margins = np.maximum(0, scores + 1 - correct_class_scores)
 margins[range(num_train), y_batch] =0
loss = np.sum(margins)/ num_train + self.reg_const * self.reg_fun(self.W)
 coeff_mat = np.zeros(( num_train, num_classes))
coeff_mat[margins > 0] = 1
coeff_mat[range(num_train), y_batch] = - np.sum(coeff_mat, axis=1)
 dW =(X_batch.T ).dot(coeff_mat)
 dW = dW/num_train + self.reg_const * self.reg_fun_grad(self.W)
 return loss, dW
def fit(self, X, y, **kwargs):
if kwargs:
self.learning_rate = kwargs['learning_rate']
self.reg_const = kwargs['reg_const']
self.l1 = kwargs['l1']
if self.l1:
self.reg_fun = lambda x: np.max(np.sum(np.abs(x), axis=0))
self.reg_fun_grad = lambda x: np.sum([np.sign(val)for val in x.flatten() if not np.isclose(val,0, atol=1e-5)])
 else:
self.reg_fun = lambda x: 0.5*np.sum(x*x)
 self.reg_fun_grad=lambda x:x
self.num_iters = kwargs['num_iters']
self.batch_size = kwargs['batch_size']
num_train, dim = X.shape
num_classes = 2
if self.W is None:
self.W = 0.001 * np.random.randn(int(dim), int(num_classes))
for it in trange(self.num_iters, mininterval=5):
batch_idx = np.random.choice(num_train, self.batch_size, replace = True)
 X_batch = X[batch_idx]
y_batch = y[batch_idx]
loss, grad = self.loss(X_batch, y_batch.astype(int))
self.W += - self.learning_rate * grad
def get_params(self, deep = False):
return {'learning_rate': self.learning_rate, 'reg_const': self.reg_const,
'l1': self.l1, 'num_iters':self.num_iters, 'batch_size':self.batch_size}
def set_params(self, **parameters):
for parameter, value in parameters.items() :
setattr(self, parameter, value)
 return self
def predict(self, X):
scores = X.dot(self.W)
 y_pred = np.argmax(scores, axis =1)
 return y_pred
def score(self, X, y):
assert X.shape[0] == y.shape[0]
correct = sum(1 for actual, prediction in zip(y, self.predict(X)) if np.sign(actual)== np.sign(prediction))
return correct/X.shape[0]
%%time
for feature in numerical_features:
method = methods[feature]
if method=='boxcox':
transform_data[feature] = boxcox1p(transform_data[feature]-transform_data[feature].min() , boxcox_normmax(transform_data[feature]-transform_data[feature].min() + 1))
elif method=='log':
transform_data[feature] = np.log1p(transform_data[feature]-transform_data[feature].min())
 elif method=='exp':
transform_data[feature] = np.exp(transform_data[feature]-transform_data[feature].min())
 elif method=='power':
transform_data[feature] =(transform_data[feature]-transform_data[feature].min())**.5
else:
pass<sort_values>
svm = SVM()
svm.fit(training_data, training_labels, learning_rate=0.01, reg_const=0.05, l1=False, num_iters=100_000, batch_size=200 )
%%time
numerical_features = ['item_cnt_month']
transform_data = matrix
transform_data[numerical_features].apply(lambda x: skew(x.dropna())).sort_values(ascending=False )<sort_values>
svm.score(testing_data, testing_labels )
%%time
transform_data[numerical_features].apply(lambda x: kurt(x.dropna())).sort_values(ascending=False )<train_model>
clf = GridSearchCV(SVM() , param_grid={'learning_rate':[0.0001, 0.0005, 0.001], 'reg_const':[0.0001,0.001,0.01,0.1], 'l1': [False], 'num_iters':[200_000], 'batch_size':[200]}, cv=5, scoring='accuracy', n_jobs=-1)
 clf.fit(testing_data, testing_labels )
print('After log1p transform, skew = ', skew(np.log1p(transform_data[numerical_features] ).dropna()))<feature_engineering>
print(clf.best_params_)
 print(clf.best_score_ )
transform_data[numerical_features] = np.log1p(transform_data[numerical_features] )<data_type_conversions>
cross_val_score(SVM() , training_data, y=training_labels, scoring='accuracy',
fit_params=clf.best_params_, cv=5, n_jobs=-1 )
test_data['date_block_num'] = 34
test_data['date_block_num'] = test_data['date_block_num'].astype(np.int8)
 test_data['shop_id'] = test_data['shop_id'].astype(np.int8)
 test_data['item_id'] = test_data['item_id'].astype(np.int16 )<concatenate>
clf.best_estimator_.score(testing_data, testing_labels )
matrix = pd.concat([matrix, test_data], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True )<data_type_conversions>
svm = clf.best_estimator_
eval_data, _ = load_svmlight_file('.. /input/movie-ratings/movie-ratings/data-splits/data.eval.anon')
 eval_data = csr_matrix(( eval_data.data, eval_data.indices, eval_data.indptr), shape=(eval_data.shape[0], training_data.shape[1]))
submission_pred = svm.predict(eval_data )
%%time
matrix = pd.merge(matrix, shop_data, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, item_data, on=['item_id'], how='left')
 matrix = pd.merge(matrix, item_category_data, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8 )<merge>
with open('submission.csv', 'w')as submission:
with open('.. /input/movie-ratings/movie-ratings/data-splits/data.eval.anon.id', 'r')as example_ids:
submission.write('example_id,label
')
 for example_id, label in zip(example_ids, submission_pred):
submission.write('{},{}
'.format(example_id.strip() , int(label)) )