Kaggle competition: Quick, Draw! Doodle Recognition Challenge
def date_adjust(df):
df[['release_month', 'release_day','release_year']] = df['release_date'].str.split('/',expand=True ).replace(np.nan,0 ).astype(int)
 releaseDate = pd.to_datetime(df['release_date'])
 df['release_week'] = releaseDate.dt.week
df['release_dayofweek'] = releaseDate.dt.dayofweek
df['release_quarter'] = releaseDate.dt.quarter
df['original_english'] = df['original_language'].apply(lambda x: 1 if x == 'en' else 0)
 df = df.drop(['collection_name','original_language','release_date'], axis=1)
 date_adjust(train)
 date_adjust(test)
 train = train.drop(['collection_name','original_language','release_date','id'], axis=1)
 test = test.drop(['collection_name','original_language','release_date','id'], axis=1 )<feature_engineering>
import tensorflow as tf
import os
def feature_transform(df):
df['budget_runtime'] =(df['budget'] + 1)/(df['runtime']+1)
 df['poularity_year'] =(df['popularity']+1)/(df['release_year']+1)
 df['budget_popularity'] =(df['budget']+1)/(df['popularity']+1)
 df['budget_year'] =(df['budget'] + 1)/(df['release_year']+1)
 df['runtime_year'] =(df['runtime']+1)/(df['release_year']+1)
 feature_transform(train)
 feature_transform(test )<prepare_x_and_y>
sub_df = pd.read_csv(data_path+'sample_submission.csv')
 print("test data 수:",len(sub_df))
sub_df.head()
train['budget'] = train['budget'].apply(np.log1p)
 test['budget'] = test['budget'].apply(np.log1p)
 X = np.asarray(train.drop(['revenue'], axis=1))
y = np.asarray(train.revenue.apply(np.log1p)) 
<train_model>
train_file_path = "/kaggle/input/quickdraw-doodle-recognition/train_raw/"
params = {'objective':'reg:linear',
'eta':0.01,
'max_depth':6,
'min_child_weight':3,
'subsample':0.8,
'colsample_bytree':0.8,
'colsample_bylevel':.5,
'gamma':1.45,
'eval_metric':'rmse',
'seed':1,
'silent':True}
def train_xgb(X_train, y_train, X_test, y_test):
xgb_data = [(xgb.DMatrix(X_train, y_train), 'train'),(xgb.DMatrix(X_test, y_test),'valid')]
xgb_model = xgb.train(params, xgb.DMatrix(X_train,y_train),
5000, xgb_data,
verbose_eval=100,
early_stopping_rounds=500)
 return xgb_model
error = []
kf = list(KFold(n_splits=10, random_state=1, shuffle=True ).split(X))
for i,(train_index, test_index)in enumerate(kf):
X_train, X_test =X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
xgb_model = train_xgb(X_train, y_train, X_test, y_test)
 error.append(xgb_model.best_score)
 print("Error: {} +/- {}".format(np.mean(error), np.std(error)))
<train_model>
train_csvs= os.listdir(train_file_path)
 print("train_raw 폴더 내 파일 수:", len(train_csvs))
print(train_csvs[:5])
 file_size = 0
label_names = []
for csv_file in train_csvs:
file_size += os.path.getsize(train_file_path + csv_file)
 label_names.append(csv_file.replace('.csv',''))
print("파일 크기 : ", file_size//(1024*1024*1024),"GB")
 label_names = sorted(label_names,key=lambda x : str.lower(x+'.csv'))
xgb_full_train = xgb.XGBRegressor(objective = 'reg:linear',
eta = 0.01,
max_depth = 6,
min_child_weight = 3,
subsample = 0.8,
colsample_bytree = 0.7,
eval_metric = 'rmse',
seed = 1,
n_estimators = 2800)
 xgb_full_train.fit(X, y)
<predict_on_test>
hold_out_set= 'train_k99'
y_predictions = xgb_full_train.predict(test.values )<save_to_csv>
import numpy as np
import json
submission = pd.read_csv(".. /input/sample_submission.csv")
 submission['revenue'] = np.exp(y_predictions)
 submission.to_csv('xgb_sub.csv',index=False )<set_options>
def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
py.init_notebook_mode(connected=True)
<load_from_csv>
def map_at3(y_true, y_pred):
map3 = tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=1)*0.5
map3 += tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=2)*0.17
map3 += tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)*0.33
return map3
or_train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 or_test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
or_train = text_to_dict(or_train)
 or_test = text_to_dict(or_test)
 train = or_train.copy()
test = or_test.copy()<categorify>
ext_data_path = '/kaggle/input/doodle-model/'
chs = ['belongs_to_collection']
for table in [train, test]:
for col in chs:
trs = table[col].apply(lambda x:sorted(list(map(lambda y:y['name'], x))))
text = trs.apply(lambda x:" ".join(x))
table[col] = text<feature_engineering>
outputs = os.listdir(ext_data_path)
 hold_out_probs = [ext_data_path+ f for f in outputs if f.find('ho_prob')>= 0 ]
test_out_probs = [ext_data_path+ f for f in outputs if f.find('test_prob')>= 0 ]
hold_out_probs = sorted(hold_out_probs)
 test_out_probs = sorted(test_out_probs)
 ho_df = pd.read_csv(ext_data_path+hold_out_set)
 ho_s = []
for prob_path in hold_out_probs:
ho = np.load(prob_path)
 ho = ho[:len(ho_df)]
ho_s.append(ho)
 targets = ho_df.y.to_numpy()
text_train = or_train.copy()
text_test = or_test.copy()
text_train=text_train.fillna('')
 text_test=text_test.fillna('')
 text_cols = ['belongs_to_collection','genres','production_companies','production_countries','Keywords','cast','crew']
for col in text_cols:
trs=text_train[col].apply(lambda x:sorted(list(map(lambda y:y['name'], x))))
text = trs.apply(lambda x:" ".join(x)+" " if len(x)>0 else "")
 text_train[col] = text.apply(lambda x:x.strip())
 trs=text_test[col].apply(lambda x:sorted(list(map(lambda y:y['name'], x))))
text = trs.apply(lambda x:" ".join(x)+" " if len(x)>0 else "")
 text_test[col] = text.apply(lambda x:x.strip() )<string_transform>
del(ho_s)
 gc.collect()
choose_text = ['belongs_to_collection','genres','production_companies','production_countries','overview']
text_train['text'] = text_train[choose_text].apply(lambda x: ''.join(x), axis=1)
 text_train = text_train['text']
text_train = text_train.apply(lambda x:x.split(' '))
text_test['text'] = text_test[choose_text].apply(lambda x: ''.join(x), axis=1)
 text_test = text_test['text']
text_test = text_test.apply(lambda x:x.split(' '))<feature_engineering>
xin = tf.keras.layers.Input(( len(label_names),ho_arr.shape[2]))
x = tf.keras.layers.Convolution1D(1,kernel_size=1,activation='linear',use_bias=False )(xin)
 x = tf.keras.layers.Reshape(target_shape=(len(label_names),))(x)
 wensemble_model = tf.keras.Model(inputs=xin, outputs=x)
 wensemble_model.summary()
embeddings_index = {}
with open(os.path.join('.. /input/glove6b50dtxt/glove.6B.50d.txt')) as f:
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs<define_variables>
check_point=tf.keras.callbacks.ModelCheckpoint(monitor='map_at3',verbose=1
,filepath='ensemble_w.h5',save_best_only=True,mode='max')
 wensemble_model.compile(optimizer=tf.keras.optimizers.Adam(1),loss='mse', metrics=[ map_at3])
 wensemble_model.fit(x=ho_arr,y=tf.keras.utils.to_categorical(targets)
 , epochs=20, batch_size=10000,verbose=1, callbacks=[check_point] )
EMBEDDING_DIM = max(max(text_train.apply(lambda x:len(x))), max(text_test.apply(lambda x:len(x)))) *50
num_words = len(text_train)
 embedding_matrix = np.zeros(( num_words, EMBEDDING_DIM))
test_matrix = np.zeros(( len(text_test), EMBEDDING_DIM))<feature_engineering>
wensemble_model.load_weights('ensemble_w.h5')
 wensemble_model.get_weights()
for row in range(len(text_train)) :
line = text_train[row]
for i in range(len(line)) :
k = line[i].lower()
if k in embeddings_index:
vec = embeddings_index[k]
embedding_matrix[row][i*50:(i+1)*50] = vec
for row in range(len(text_test)) :
line = text_test[row]
for i in range(len(line)) :
k = line[i].lower()
if k in embeddings_index:
vec = embeddings_index[k]
test_matrix[row][i*50:(i+1)*50] = vec<choose_model_class>
del(ho_arr)
 gc.collect()
input_img = Input(shape=(EMBEDDING_DIM,), name='input_img')
 encode1 = Dense(EMBEDDING_DIM//2, activation='relu', name='encode1' )(input_img)
 hidden = Dropout(0.5 )(encode1)
 encode2 = Dense(128, activation='relu', name='encode2' )(hidden)
 encode3 = Dense(32, activation='relu', name='encode3' )(encode2)
 bottleneck = Dense(8, activation='relu', name='bottleneck' )(encode3)
 decode1 = Dense(32, activation='relu', name='decode1' )(bottleneck)
 decode2 = Dense(128, activation='relu', name='decode2' )(decode1)
 decode3 = Dense(EMBEDDING_DIM//2, activation='relu', name='decode3' )(decode2)
 hidden = Dropout(0.5 )(decode3)
 decode4 = Dense(EMBEDDING_DIM, activation='relu', name='decode4' )(decode3)
 ae = models.Model(input_img, decode4 )<train_model>
res = np.array(wensemble_model.get_weights() ).squeeze()
learning_rate = 1E-3
ae.compile(loss='mean_squared_error', optimizer=optimizers.RMSprop(lr=learning_rate))
history = ae.fit(x_tr, x_tr,
batch_size=128,
epochs=20,
validation_data=(x_val, x_val))<predict_on_test>
ens_prob = np.zeros(( len(sub_df),len(label_names)))
 for i, prob_path in enumerate(test_out_probs):
ens_prob +=(np.load(prob_path)* res[i] )
ae_encoder = models.Model(input_img, bottleneck)
 encoded_test = ae_encoder.predict(test_matrix)
 encoded_test
encoded_train = ae_encoder.predict(embedding_matrix)
 encoded_train<predict_on_test>
def balancing_predictions(test_prob, factor = 0.1, minfactor = 0.001, patient = 5, permit_cnt=332, max_search=10000, label_num=340):
maxk = float('inf')
 s_cnt = np.zeros(label_num)
 for i in range(max_search):
ctop1 = Counter(np.argmax(test_prob,axis=1))
ctop1 = sorted(ctop1.items() , key=itemgetter(1), reverse=True)
 if maxk > ctop1[0][1]:
maxk = ctop1[0][1]
else:
s_cnt[ctop1[0][0]]+=1
if np.max(s_cnt)>patient:
if factor< minfactor:
print('stop min factor')
 break
s_cnt=np.zeros(label_num)
 factor*=0.99
print('reduce factor: ', factor, ', current max category num: ', ctop1[0][1])
 if ctop1[0][1] <= permit_cnt:
print('idx: ',ctop1[0][0] ,', num: ', ctop1[0][1])
 break
test_prob[:,ctop1[0][0]] *=(1.0-factor)
 return test_prob
<compute_test_metric><EOS> 
bal_test_prob = balancing_predictions(ens_prob)
 bal_top3 = preds2catids(bal_test_prob)
 id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(label_names)}
bal_top3cats = bal_top3.replace(id2cat)
 sub_df['word'] = bal_top3cats['a'] + ' ' + bal_top3cats['b'] + ' ' + bal_top3cats['c']
bal_submission = sub_df[['key_id', 'word']]
bal_submission.to_csv('submission_bal_ens.csv', index=False )