Kaggle competition: What's Cooking? (Kernels Only)
<SOS> metric: CategorizationAccuracy Kaggle data source: whats-cooking?-(kernels-only)<define_variables>
%matplotlib inline
features = [
'lagtime',
'lagtime_mean',
'content_id',
'task_container_id',
'user_lecture_cumsum',
'user_lecture_lv',
'prior_question_elapsed_time',
'delta_prior_question_elapsed_time',
'user_correctness',
'user_correct_cumcount',
'user_correct_cumsum',
'content_correctness',
'content_correctness_std',
'content_count',
'content_sum',
'task_container_correctness',
'task_container_std',
'task_container_sum',
'bundle_correctness',
'attempt_no',
'part',
'part_correctness_mean',
'part_correctness_std',
'tags1',
'tags1_correctness_mean',
'tags1_correctness_std',
'bundle_id',
'part_bundle_id',
'explanation_mean',
'explanation_cumsum',
'prior_question_had_explanation',
]
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags1',
'bundle_id',
'part_bundle_id',
'prior_question_had_explanation',
] 
<init_hyperparams>
tokenize = TweetTokenizer().tokenize
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 350,
'max_bin':700,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.58,
'bagging_fraction': 0.58,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.05,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df.sample(n=1200*10000)
 print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.025)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 valid_df=train_df_clf.sample(frac=0.15)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('valid_df length：',len(valid_df)) 
<drop_column>
print(os.listdir())
 train = pd.read_json('/kaggle/input/whats-cooking-kernels-only/train.json')
 test = pd.read_json('/kaggle/input/whats-cooking-kernels-only/test.json')
 print(train.head())
 ytrain = train['cuisine']
print(ytrain.head(5))
Id = test['id']
print(Id.head(5))
del train_df_clf
del valid_df
gc.collect()<prepare_x_and_y>
tfidf = TfidfVectorizer(binary=True )
for i in range(0,num):
tr_data = lgb.Dataset(trains[i][features], label=trains[i][target])
 va_data = lgb.Dataset(valids[i][features], label=valids[i][target])
 del trains
del valids
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 lgb.plot_importance(model, importance_type='gain')
 plt.show()
del tr_data
del va_data
gc.collect() 
<data_type_conversions>
train2 = train
print(( train2['ingredients'][0]))
print(arraytotext(train2['ingredients'][0]))
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
del user_agg
del content_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()<data_type_conversions>
train_features = tfidf.fit_transform(arraytotext(train['ingredients']))
test_features = tfidf.transform(arraytotext(test['ingredients']))
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
lagtime_mean_dict = lagtime_agg['mean'].astype('int32' ).to_dict(defaultdict(int))
del user_lecture_agg
del lagtime_agg
gc.collect()<categorify>
classifier = SVC(C=200, kernel='rbf', degree=3,gamma=1, \
coef0=1, shrinking=True,tol=0.001, probability=False,\
cache_size=200,class_weight=None, verbose=False,\
max_iter=-1,decision_function_shape=None,\
random_state=None )
attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()<categorify>
model = OneVsRestClassifier(classifier)
 scores = cross_val_score(classifier,train_features, ytrain, cv=2)
 print("Accuracy: %0.2f(+/- %0.2f)" % \ 
(scores.mean() , scores.std() * 2))
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del user_prior_question_elapsed_time
gc.collect()<feature_engineering>
model.fit(train_features, ytrain )
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]<choose_model_class>
predictions = model.predict(test_features)
 print(predictions )
<split><EOS> 
submission = pd.DataFrame()
submission['id'] = Id
submission['cuisine'] = predictions
submission.to_csv('submission.csv', index=False)