Kaggle competition: What's Cooking? (Kernels Only)
<SOS> metric: CategorizationAccuracy Kaggle data source: whats-cooking?-(kernels-only)<merge>
tqdm.pandas()
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how="left")
 test_df['part_1'].fillna(0, inplace = True)
 test_df['part_2'].fillna(0, inplace = True)
 test_df['part_3'].fillna(0, inplace = True)
 test_df['part_4'].fillna(0, inplace = True)
 test_df['part_5'].fillna(0, inplace = True)
 test_df['part_6'].fillna(0, inplace = True)
 test_df['part_7'].fillna(0, inplace = True)
 test_df['type_of_concept'].fillna(0, inplace = True)
 test_df['type_of_intention'].fillna(0, inplace = True)
 test_df['type_of_solving_question'].fillna(0, inplace = True)
 test_df['type_of_starter'].fillna(0, inplace = True)
 test_df['part_1_boolean'].fillna(0, inplace = True)
 test_df['part_2_boolean'].fillna(0, inplace = True)
 test_df['part_3_boolean'].fillna(0, inplace = True)
 test_df['part_4_boolean'].fillna(0, inplace = True)
 test_df['part_5_boolean'].fillna(0, inplace = True)
 test_df['part_6_boolean'].fillna(0, inplace = True)
 test_df['part_7_boolean'].fillna(0, inplace = True)
 test_df['type_of_concept_boolean'].fillna(0, inplace = True)
 test_df['type_of_intention_boolean'].fillna(0, inplace = True)
 test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)
 test_df['type_of_starter_boolean'].fillna(0, inplace = True)
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['avg_question_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 full_preds = full_model.predict_proba(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_question_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])[:, 1]
full_preds_xgb = full_xgb.predict_proba(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_question_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])[:, 1]
full_preds_lr = full_lr.predict_proba(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_question_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])[:, 1]
X_test = rfe.transform(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_question_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])
 preds = [model.predict_proba(X_test)[:,1] for model in models]
predictions = preds[0]
for i in range(1, 5):
predictions += preds[i]
predictions /= 5
test_df['answered_correctly'] = predictions * 0.75 + full_preds * 0.125 + full_preds_xgb * 0.75 + full_preds_lr * 0.05
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<load_from_csv>
train = pd.read_json('.. /input/train.json')
 test = pd.read_json('.. /input/test.json' )
dir = '.. /input/e6-oof-prediction//'
np.random.seed(42)
 oof_one = pd.read_csv(dir + 'oof_e6_exp1_seed_42.csv')
 test_one = pd.read_csv(dir + 's_e6_exp1_seed_42.csv')
 oof_one = oof_one.sort_values(by=['image_name'],
ascending=True ).reset_index(drop=True)
 test_one = test_one.sort_values(by=['image_name'],
ascending=True ).reset_index(drop=True)
 oof_two = pd.read_csv(dir + 'oof_e6_exp2_seed_42.csv')
 test_two = pd.read_csv(dir + 's_e6_exp2_seed_42.csv')
 oof_two = oof_two.sort_values(by=['image_name'],
ascending=True ).reset_index(drop=True)
 test_two = test_two.sort_values(by=['image_name'],
ascending=True ).reset_index(drop=True)
 oof_three = pd.read_csv(dir + 'oof_e6_exp3_seed_42.csv')
 test_three = pd.read_csv(dir + 's_e6_exp3_seed_42.csv')
 oof_three = oof_three.sort_values(by=['image_name'],
ascending=True ).reset_index(drop=True)
 test_three = test_three.sort_values(by=['image_name'],
ascending=True ).reset_index(drop=True)
 oof_four = pd.read_csv(dir + 'oof_e6_exp4_seed_42.csv')
 test_four = pd.read_csv(dir + 's_e6_exp4_seed_42.csv')
 oof_four = oof_four.sort_values(by=['image_name'],
ascending=True ).reset_index(drop=True)
 test_four = test_four.sort_values(by=['image_name'],
ascending=True ).reset_index(drop=True )<count_values>
train['num_ingredients'] = train['ingredients'].apply(len)
 train = train[train['num_ingredients'] > 1]
oof_one.fold.value_counts()<prepare_output>
lemmatizer = WordNetLemmatizer()
def preprocess(ingredients):
ingredients_text = ' '.join(ingredients)
 ingredients_text = ingredients_text.lower()
ingredients_text = ingredients_text.replace('-', ' ')
 words = []
for word in ingredients_text.split() :
if re.findall('[0-9]', word): continue
if len(word)<= 2: continue
if '’' in word: continue
word = lemmatizer.lemmatize(word)
 if len(word)> 0: words.append(word)
 return ' '.join(words)
 for ingredient, expected in [ 
('Eggs', 'egg'), 
('all-purpose flour', 'all purpose flour'), 
('purée', 'purée'), 
('1% low-fat milk', 'low fat milk'), 
('half & half', 'half half'), 
('safetida(powder)', 'safetida(powder)')
 ]:
actual = preprocess([ingredient])
 assert actual == expected, f'"{expected}" is excpected but got "{actual}"'
blend_train = []
blend_test = []
blend_train.append(oof_one.pred)
 blend_train.append(oof_two.pred)
 blend_train.append(oof_three.pred)
 blend_train.append(oof_four.pred)
 blend_train = np.array(blend_train)
 blend_test.append(test_one.target)
 blend_test.append(test_two.target)
 blend_test.append(test_three.target)
 blend_test.append(test_four.target)
 blend_test = np.array(blend_test )<compute_train_metric>
train['x'] = train['ingredients'].progress_apply(preprocess)
 test['x'] = test['ingredients'].progress_apply(preprocess)
 train.head()
def roc_min_func(weights):
final_prediction = 0
for weight, prediction in zip(weights, blend_train):
final_prediction += weight * prediction
return roc_auc_score(np.array(oof_one.target), final_prediction)
 print('
Finding Blending Weights...')
 res_list = []
weights_list = []
for k in range(100):
starting_values = np.random.uniform(size=len(blend_train))
bounds = [(0, 1)] * len(blend_train)
 res = minimize(roc_min_func,
starting_values,
method='L-BFGS-B',
bounds=bounds,
options={'disp': False,
'maxiter': 100000})
 res_list.append(res['fun'])
 weights_list.append(res['x'])
 print('{iter}\tScore: {score}\tWeights: {weights}'.format(
 iter=(k + 1),
score=res['fun'],
weights='\t'.join([str(item)for item in res['x']])))
 bestSC = np.max(res_list)
 bestWght = weights_list[np.argmax(res_list)]
weights = bestWght
blend_score = round(bestSC, 6 )<train_model>
vectorizer = make_pipeline(
 TfidfVectorizer(sublinear_tf=True),
FunctionTransformer(lambda x: x.astype('float16'), validate=False)
)
 x_train = vectorizer.fit_transform(train['x'].values)
 x_train.sort_indices()
x_test = vectorizer.transform(test['x'].values )
print('
Ensemble Score: {best_score}'.format(best_score=bestSC))
print('
Best Weights: {weights}'.format(weights=bestWght))
train_prices = np.zeros(len(blend_train[0]))
test_prices = np.zeros(len(blend_test[0]))
print('
Your final model:')
 for k in range(len(blend_test)) :
print(' %.6f * model-%d' %(weights[k],(k + 1)))
 test_prices += blend_test[k] * weights[k]
for k in range(len(blend_train)) :
train_prices += blend_train[k] * weights[k]<feature_engineering>
label_encoder = LabelEncoder()
y_train = label_encoder.fit_transform(train['cuisine'].values)
 dict(zip(label_encoder.classes_, label_encoder.transform(label_encoder.classes_)) )
test_one.target =(test_one.target.values*bestWght[0] +
test_two.target.values*bestWght[1] +
test_three.target.values*bestWght[2]+
test_four.target.values*bestWght[3])/max(bestWght )<load_from_csv>
estimator = SVC(
 C=50,
kernel='rbf',
gamma=1.4,
coef0=1,
cache_size=500, 
)
 classifier = OneVsRestClassifier(estimator, n_jobs=-1 )
train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')
 test = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv' )<load_from_csv>
%%time
scores = cross_validate(classifier, x_train, y_train, cv=3)
 scores['test_score'].mean()
dirname = '/kaggle/input/e6-oof-prediction'
prefix = ['exp1', 'exp2', 'exp3', 'exp4', ]
for model in range(4):
_oof = pd.read_csv(os.path.join(dirname, f"oof_e6_{prefix[model]}_seed_42.csv"))
score = roc_auc_score(_oof['target'], _oof['pred'])
 print(f"{prefix[model]}: OOF auc:{score:.4}")
 _oof = _oof.rename(columns={"pred":prefix[model]} ).drop(["target"],axis=1)
 if "fold" in _oof.columns:
_oof = _oof.drop(["fold"],axis=1)
 train = train.merge(_oof, on="image_name")
 _sub = pd.read_csv(os.path.join(dirname, f"s_e6_{prefix[model]}_seed_42.csv"))
_sub.columns = ["image_name", prefix[model]]
test = test.merge(_sub, on="image_name" )<compute_test_metric>
%%time
classifier.fit(x_train, y_train )
def bo_pred(df):
x =(c0*df[ prefix[0] ] +
c1*df[ prefix[1] ] +
c2*df[ prefix[2] ] +
c3*df[ prefix[3] ])
 return x
train["pred"] = bo_pred(train)
 score = roc_auc_score(train['target'], train['pred'])
 print(f"auc bo:{score}" )<load_from_csv>
y_pred = label_encoder.inverse_transform(classifier.predict(x_train))
y_true = label_encoder.inverse_transform(y_train)
 print(f'accuracy score on train data: {accuracy_score(y_true, y_pred)}')
 def report2dict(cr):
rows = []
for row in cr.split("
"):
parsed_row = [x for x in row.split(" ")if len(x)> 0]
if len(parsed_row)> 0: rows.append(parsed_row)
 measures = rows[0]
classes = defaultdict(dict)
 for row in rows[1:]:
class_label = row[0]
for j, m in enumerate(measures):
classes[class_label][m.strip() ] = float(row[j + 1].strip())
 return classes
report = classification_report(y_true, y_pred)
 pd.DataFrame(report2dict(report)).T
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(10,4))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(10,4))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()<define_variables>
y_pred = label_encoder.inverse_transform(classifier.predict(x_test))
test['cuisine'] = y_pred
test[['id', 'cuisine']].to_csv('submission.csv', index=False)
 test[['id', 'cuisine']].head()