Kaggle competition: National Data Science Challenge 2019 - Advanced
<SOS> metric: map@{k} Kaggle data source: national-data-science-challenge-2019-advanced<count_unique_values>
from pathlib import Path
import json
import re
import pandas as pd
import numpy as np
from tqdm import tqdm_notebook
len(csse_data['Province_State'].unique() )<count_unique_values>
DATA_DIR = Path('.. /input/ndsc-advanced')
 BEAUTY_TRAIN_CSV = DATA_DIR / 'beauty_data_info_train_competition.csv'
FASHION_TRAIN_CSV = DATA_DIR / 'fashion_data_info_train_competition.csv'
MOBILE_TRAIN_CSV = DATA_DIR / 'mobile_data_info_train_competition.csv'
BEAUTY_TEST_CSV = DATA_DIR / 'beauty_data_info_val_competition.csv'
FASHION_TEST_CSV = DATA_DIR / 'fashion_data_info_val_competition.csv'
MOBILE_TEST_CSV = DATA_DIR / 'mobile_data_info_val_competition.csv'
LAST_SUBMITTED_CSV = Path('.. /input/aoeul-solution-step-3-linearsvc-dl-model/Ensembled_SVC_DL_predictions.csv' )
len(dftrain['Province_State'].unique() )<merge>
beauty_train_df = pd.read_csv(BEAUTY_TRAIN_CSV)
 fashion_train_df = pd.read_csv(FASHION_TRAIN_CSV)
 mobile_train_df = pd.read_csv(MOBILE_TRAIN_CSV)
 beauty_test_df = pd.read_csv(BEAUTY_TEST_CSV)
 fashion_test_df = pd.read_csv(FASHION_TEST_CSV)
 mobile_test_df = pd.read_csv(MOBILE_TEST_CSV)
 prev_subm_df = pd.read_csv(LAST_SUBMITTED_CSV )
dftrain = dftrain.join(confirmed[['Country_Region', 'Province_State', 'Lat', 'Long']].set_index(['Province_State', 'Country_Region']), on=['Province_State', 'Country_Region'] )<feature_engineering>
categories = ['beauty', 'fashion', 'mobile']
train_dfs = [beauty_train_df, fashion_train_df, mobile_train_df]
test_dfs = [beauty_test_df, fashion_test_df, mobile_test_df]
dftrain['Dayofyear'] = dftrain['Date'].dt.dayofyear
dftest['Dayofyear'] = dftest['Date'].dt.dayofyear<data_type_conversions>
matched_label_df = pd.DataFrame()
for cat, train_df, test_df in zip(categories, train_dfs, test_dfs):
print(f'Matching {cat} training and test data...')
 df_train = train_df.rename(columns={'title': 'title_train', 'itemid': 'itemid_train'})
 df_test = test_df[['itemid', 'title']].rename(columns={'title': 'title_test', 'itemid': 'itemid_test'})
 df_combined = pd.merge(df_train, df_test, how='inner', left_on=['title_train'], right_on=['title_test'])
 df_combined.drop(['image_path','itemid_train', 'title_train'], axis=1, inplace=True)
 df_clean = df_combined.drop_duplicates()
print('Finding test data with exact same title as train data with unique labels')
 single_instance=[]
for title in tqdm_notebook(df_clean.title_test.unique()):
if(len(df_clean[df_clean.title_test==title])== 1):
single_instance.append(title)
 print(f'{len(single_instance)} test-train title pair matches found in {cat} category')
 single_inst_df = df_clean[df_clean.title_test.isin(single_instance)]
feat_cols = single_inst_df.columns.drop(['itemid_test', 'title_test'])
 id_label_list=[]
for _, row in single_inst_df.iterrows() :
for feat in feat_cols:
if not np.isnan(row[feat]):
itemid = '_'.join([str(row['itemid_test']), str(feat)])
 answer = str(int(row[feat]))
id_label_list.append(( itemid, answer))
prev_subm_df['first_pred'] = prev_subm_df['tagging'].apply(lambda x: x.split(' ')[0])
 list_new_id = []
list_new_ans = []
for id_ans in tqdm_notebook(id_label_list):
subm_first_label = int(prev_subm_df.loc[prev_subm_df.id==id_ans[0]]['first_pred'])
 if subm_first_label != int(id_ans[1]):
print(f'Test id: {id_ans[0]:28} Submitted label: {subm_first_label:<5} Train data label: {id_ans[1]}')
 list_new_id.append(id_ans[0])
 list_new_ans.append(id_ans[1])
 label_df = pd.DataFrame(
 {'id': list_new_id, 'tagging': list_new_ans},
columns = ['id', 'tagging'])
 matched_label_df = pd.concat([matched_label_df, label_df], axis=0)
 print()
<feature_engineering><EOS> 
for _, row in tqdm_notebook(matched_label_df.iterrows()): 
(old_value,)= prev_subm_df.loc[prev_subm_df.id == row.id, 'tagging']
prev_subm_df.loc[prev_subm_df.id == row.id, 'tagging'] = str(row.tagging)+ ' ' + str(old_value.split(' ')[0])
 prev_subm_df.drop(['first_pred'], axis=1, inplace=True)
 prev_subm_df.to_csv('Test_Matched_Train_predictions.csv', index=False )