Kaggle competition: Fashion MNIST challenge201907
<SOS> metric: categorizationaccuracy Kaggle data source: fashion-mnist-challenge201907<merge>
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
pl.seed = 0
N = 1500
probs = train_d.landmark_id.value_counts() / train_d.shape[0]
probs = probs.iloc[:N]
probs = pd.DataFrame({'landmark_id': probs.index,
'probability': probs.values}, index=pl.arange(N))
T = pd.merge(train_d, probs, on='landmark_id', how='inner')
 inx = pl.randint(0, T.shape[0], submission.shape[0])
 submission['landmark_id'] = T.landmark_id.iloc[inx].values
submission['prob'] = T.probability.iloc[inx].values
submission['landmarks'] = submission.landmark_id.astype(str)+ ' ' + submission.prob.astype(str)
 submission[['id','landmarks']].head()
submission[['id','landmarks']].to_csv('submission_outer.csv', index=False )<set_options>
import keras
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.layers.normalization import BatchNormalization
%matplotlib inline<load_from_csv>
data_train = pd.read_csv('.. /input/train42000.csv')
 data_test = pd.read_csv('.. /input/test28000.csv' )
base_dir = '.. /input/'
train_df = pd.read_csv(os.path.join(base_dir, 'train.csv'))
test_df = pd.read_csv(os.path.join(base_dir, 'test.csv'))
train_df.sample(3 )<load_from_csv>
X = np.array(data_train.iloc[:, 1:] )
submit_df = pd.read_csv(os.path.join(base_dir, 'sample_submission.csv'))
def_guess = train_df['landmark_id'].value_counts() /train_df['landmark_id'].value_counts().sum()
def_guess.index[0], def_guess.values[0]<save_to_csv>
y_tmp = np.array(data_train.iloc[:, 0])
 y_tmp
submit_df['landmarks'] = '%d %2.2f' %(def_guess.index[0], def_guess.values[0])
 submit_df.to_csv('submission.csv', index=False)
 submit_df.sample(2 )<randomize_order>
y = to_categorical(y_tmp)
 y
np.random.seed(2018)
 r_idx = lambda : np.random.choice(def_guess.index, p = def_guess.values )<save_to_csv>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=13 )
%%time
r_score = lambda idx: '%d %2.4f' %(def_guess.index[idx], def_guess.values[idx])
 submit_df['landmarks'] = submit_df.id.map(lambda _: r_score(r_idx()))
submit_df.to_csv('rand_submission.csv', index=False)
 submit_df.sample(2 )<load_from_csv>
X_test = np.array(data_test.iloc[:, 0:] )
train = pd.read_csv('.. /input/train.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 pl.seed(0)
<save_to_csv>
img_rows, img_cols = 28, 28
input_shape =(img_rows, img_cols, 1)
 X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
 X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
 X_val = X_val.reshape(X_val.shape[0], img_rows, img_cols, 1 )
probs = train.landmark_id.value_counts() / train.shape[0]
sub['landmark_id'] = train.landmark_id.iloc[pl.randint(0, train.shape[0], sub.shape[0])].values
sub['prob'] = probs[sub.landmark_id].values
sub['landmarks'] = sub.landmark_id.astype(str)+ ' ' + sub.prob.astype(str)
 sub[['id','landmarks']].to_csv('submission_1.csv', index=False)
 sub[['id','landmarks']].head()<save_to_csv>
X_train = X_train.astype('float32')
 X_test = X_test.astype('float32')
 X_val = X_val.astype('float32' )
sub['landmarks'] = '%d %g' %(probs.index[0], probs[probs.index[0]])
 sub[['id','landmarks']].to_csv('submission_2.csv', index=False)
 sub[['id','landmarks']].head()<save_to_csv>
X_train /= 255
X_test /= 255
X_val /= 255
N = 1000
probs = train.landmark_id.value_counts() / train.shape[0]
probs = probs.iloc[:N]
probs = pd.DataFrame({'landmark_id': probs.index,
'probability': probs.values}, index=pl.arange(N))
T = pd.merge(train, probs, on='landmark_id', how='inner')
 inx = pl.randint(0, T.shape[0], sub.shape[0])
 sub['landmark_id'] = T.landmark_id.iloc[inx].values
sub['prob'] = T.probability.iloc[inx].values
sub['landmarks'] = sub.landmark_id.astype(str)+ ' ' + sub.prob.astype(str)
 sub[['id','landmarks']].to_csv('submission_3.csv', index=False)
 sub[['id','landmarks']].head()<save_to_csv>
num_classes = 10
model = Sequential()
model.add(Conv2D(20, kernel_size=(5, 5),
activation='relu',
kernel_initializer='he_normal',
input_shape=input_shape))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Flatten())
 model.add(Dense(20, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
N = 1000
probs = train.landmark_id.value_counts() / train.shape[0]
probs = probs.iloc[:N]
probs = pd.DataFrame({'landmark_id': probs.index,
'probability': 0.1}, index=pl.arange(N))
T = pd.merge(train, probs, on='landmark_id', how='inner')
 inx = pl.randint(0, T.shape[0], sub.shape[0])
 sub['landmark_id'] = T.landmark_id.iloc[inx].values
sub['prob'] = T.probability.iloc[inx].values
sub['landmarks'] = sub.landmark_id.astype(str)+ ' ' + sub.prob.astype(str)
 sub[['id','landmarks']].to_csv('submission_4.csv', index=False)
 sub[['id','landmarks']].head()<load_from_csv>
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam() ,
metrics=['accuracy'] )
train = pd.read_csv('.. /input/train.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 N = 2
probs = train.landmark_id.value_counts() / train.shape[0]
probs = probs.iloc[:N]
probs = pd.DataFrame({'landmark_id': probs.index,
'probability': 0.1}, index=pl.arange(N))
T = pd.merge(train, probs, on='landmark_id', how='inner')
 inx = pl.randint(0, T.shape[0], sub.shape[0])
 sub['landmark_id'] = T.landmark_id.iloc[inx].values
sub['prob'] = T.probability.iloc[inx].values
sub['landmarks'] = sub.landmark_id.astype(str)+ ' ' + sub.prob.astype(str)
 sub[['id','landmarks']].to_csv('submission_5.csv', index=False)
 sub[['id','landmarks']].head()<load_from_csv>
batch_size = 256
epochs = 5
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )<filter>
history = model.fit(X_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(X_val, y_val))
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]<feature_engineering>
predicted_classes = model.predict_classes(X_test )
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median<feature_engineering>
df = pd.DataFrame(predicted_classes,columns=['label'] )
<categorify><EOS> 
df.to_csv(path_or_buf="Submission.csv",index_label="image_id" )