Kaggle competition: Car Classification(Project Vision)
<SOS> metric: CategorizationAccuracy Kaggle data source: car-classificationproject-vision<drop_column>
%matplotlib inline
submission = reduce_mem_usage(submission, True)
 sales_df = reduce_mem_usage(sales_df, True)
 calendar_df = reduce_mem_usage(calendar_df, True)
 prices_df = reduce_mem_usage(prices_df, True )<define_variables>
!pip install -q -U tf-hub-nightly
!pip install -q tfds-nightly
sales_df.sample(3 )<define_variables>
train_dir = '/kaggle/input/car-classificationproject-vision/Train/Train/'
test_dir = '/kaggle/input/car-classificationproject-vision/Test/Test/Test1/'
calendar_df.head(5 )<count_values>
car_map = pd.read_csv('/kaggle/input/car-classificationproject-vision/car_map.csv')
 car_map=car_map.set_index('Cars')
 car_map_dict=car_map.to_dict('index' )
calendar_df.event_name_1.value_counts()<count_values>
batch_size = 64
IMG_HEIGHT = 224
IMG_WIDTH = 224
calendar_df.event_type_1.value_counts()<count_values>
train_image_generator = ImageDataGenerator(rescale=1./255, validation_split=0.2, horizontal_flip=True, rotation_range=5, shear_range=0.2, zoom_range=0.2, width_shift_range=0.2, height_shift_range=0.2)
 validation_image_generator = ImageDataGenerator(rescale=1./255)
calendar_df.event_name_2.value_counts()<count_values>
def oh_to_class(Y):
result = np.zeros(( Y.shape[0], 1))
for i in range(Y.shape[0]):
result[i]=np.argmax(Y[i])
 return result
calendar_df.event_type_2.value_counts()<feature_engineering>
train_data_gen = train_image_generator.flow_from_directory(batch_size=batch_size,
directory=train_dir,
shuffle=True,
target_size=(IMG_HEIGHT, IMG_WIDTH),
class_mode='categorical')
sales_df['sum_for_peariod'] = sales_df.iloc[:, 6:].sum(axis=1 )<filter>
test_dir='/kaggle/input/car-classificationproject-vision/Test/Test/'
sales_df[sales_df['sum_for_peariod'] == sales_df['sum_for_peariod'].min() ]<filter>
test_data_gen = train_image_generator.flow_from_directory(batch_size=batch_size,
directory=test_dir,
shuffle=False,
target_size=(IMG_HEIGHT, IMG_WIDTH),
class_mode=None)
sales_df[sales_df['sum_for_peariod'] == sales_df['sum_for_peariod'].max() ]<groupby>
sample_training_images, labels = next(train_data_gen)
sales_df.groupby('cat_id' ).sum_for_peariod.sum()<groupby>
train_data_gen.class_indices
sales_df.groupby(['cat_id', 'store_id'] ).sum_for_peariod.agg([min, max, np.mean, np.std] )<concatenate>
def output_labels_to_map_labels(Y, output_label_dict, map_label_dict):
result = np.zeros(Y.shape)
 for i in range(Y.shape[0]):
curr_label_ind = Y[i][0]
car_name=0
for key, val in output_label_dict.items() :
if val == curr_label_ind:
car_name = key
break
print
req_label_index = map_label_dict[car_name]['Class Numbers']
result[i][0]=req_label_index
return result
events_days = pd.concat([sales_df.loc[:, :'state_id'],
sales_df[calendar_df[:1913][~calendar_df.event_type_1[:1913].isna() ].d]],
axis = 1)
 events_days.sample(5 )<concatenate>
output_labels_to_map_labels(oh_to_class(labels[10:15]),train_data_gen.class_indices, car_map_dict )
not_events_days = pd.concat([sales_df.loc[:, :'state_id'],
sales_df[calendar_df[:1913][(calendar_df.event_type_1[:1913].isna())&(calendar_df.event_type_2[:1913].isna())].d]],
axis = 1)
 not_events_days.sample(5 )<groupby>
feature_extractor_url = "https://tfhub.dev/tensorflow/resnet_50/feature_vector/1"
events_days.groupby(['store_id', 'cat_id'] ).sum()<groupby>
feature_extractor_layer = hub.KerasLayer(feature_extractor_url,
input_shape=(224,224,3))
not_events_days.groupby(['store_id', 'cat_id'] ).sum()<concatenate>
model.compile(
 optimizer=tf.keras.optimizers.Adam() ,
loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),
metrics=['acc'])
snapCA = pd.concat([not_events_days.loc[not_events_days.state_id=='CA', :'state_id'],
not_events_days.loc[not_events_days.state_id=='CA', calendar_df[:1913][(calendar_df[:1913].snap_CA==1)&(calendar_df.event_type_1[:1913].isna())].d]],
axis = 1)
 snapTX = pd.concat([not_events_days.loc[not_events_days.state_id=='TX', :'state_id'],
not_events_days.loc[not_events_days.state_id=='TX', calendar_df[:1913][(calendar_df[:1913].snap_TX==1)&(calendar_df.event_type_1[:1913].isna())].d]],
axis = 1)
 snapWI = pd.concat([not_events_days.loc[not_events_days.state_id=='WI', :'state_id'],
not_events_days.loc[not_events_days.state_id=='WI', calendar_df[:1913][(calendar_df[:1913].snap_WI==1)&(calendar_df.event_type_1[:1913].isna())].d]],
axis = 1)
 not_snapCA = pd.concat([not_events_days.loc[not_events_days.state_id=='CA', :'state_id'],
not_events_days.loc[not_events_days.state_id=='CA', calendar_df[:1913][(calendar_df[:1913].snap_CA==0)&(calendar_df.event_type_1[:1913].isna())].d]],
axis = 1)
 not_snapTX = pd.concat([not_events_days.loc[not_events_days.state_id=='TX', :'state_id'],
not_events_days.loc[not_events_days.state_id=='TX', calendar_df[:1913][(calendar_df[:1913].snap_TX==0)&(calendar_df.event_type_1[:1913].isna())].d]],
axis = 1)
 not_snapWI = pd.concat([not_events_days.loc[not_events_days.state_id=='WI', :'state_id'],
not_events_days.loc[not_events_days.state_id=='WI', calendar_df[:1913][(calendar_df[:1913].snap_WI==0)&(calendar_df.event_type_1[:1913].isna())].d]],
axis = 1 )<filter>
class CollectBatchStats(tf.keras.callbacks.Callback):
def __init__(self):
self.batch_losses = []
self.batch_acc = []
def on_train_batch_end(self, batch, logs=None):
self.batch_losses.append(logs['loss'])
 self.batch_acc.append(logs['acc'])
 self.model.reset_metrics()
prices_df[(prices_df.item_id == 'HOBBIES_1_001')&(prices_df.store_id == 'TX_1')].wm_yr_wk.values<groupby>
steps_per_epoch = 4050//64
batch_stats_callback = CollectBatchStats()
history = model.fit_generator(train_data_gen, epochs=360,
steps_per_epoch=steps_per_epoch,
callbacks = [batch_stats_callback])
sales_df[(sales_df.item_id == 'HOBBIES_1_001')&(sales_df.store_id=='CA_1')].groupby('store_id' ).sum()<set_options>
model.evaluate(train_data_gen )
pd.options.display.max_columns = 50
CAL_DTYPES={"event_name_1": "category", "event_name_2": "category", "event_type_1": "category",
"event_type_2": "category", "weekday": "category", 'wm_yr_wk': 'int16', "wday": "int16",
"month": "int16", "year": "int16", "snap_CA": "float32", 'snap_TX': 'float32', 'snap_WI': 'float32' }
PRICE_DTYPES = {"store_id": "category", "item_id": "category", "wm_yr_wk": "int16","sell_price":"float32" }
FIRST_DAY = 800
h = 28
max_lags = 366
tr_last = 1913
fday = datetime(2016,4, 25)
 fday<load_from_csv>
train_predicted_oh = model.predict(train_data_gen )
def create_dt(is_train = True, nrows = None, first_day = 1200):
prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv", dtype = PRICE_DTYPES)
 for col, col_dtype in PRICE_DTYPES.items() :
if col_dtype == "category":
prices[col] = prices[col].cat.codes.astype("int16")
 prices[col] -= prices[col].min()
cal = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv", dtype = CAL_DTYPES)
 cal["date"] = pd.to_datetime(cal["date"])
 for col, col_dtype in CAL_DTYPES.items() :
if col_dtype == "category":
cal[col] = cal[col].cat.codes.astype("int16")
 cal[col] -= cal[col].min()
start_day = max(1 if is_train else tr_last-max_lags, first_day)
 numcols = [f"d_{day}" for day in range(start_day,tr_last+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 dt = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_validation.csv",
nrows = nrows, usecols = catcols + numcols, dtype = dtype)
 for col in catcols:
if col != "id":
dt[col] = dt[col].cat.codes.astype("int16")
 dt[col] -= dt[col].min()
if not is_train:
for day in range(tr_last+1, tr_last+ 2*h +1):
dt[f"d_{day}"] = np.nan
dt = pd.melt(dt,
id_vars = catcols,
value_vars = [col for col in dt.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales")
 dt = dt.merge(cal, on= "d", copy = False)
 dt = dt.merge(prices, on = ["store_id", "item_id", "wm_yr_wk"], copy = False)
 return dt<categorify>
test_predicted_oh = model.predict_generator(test_data_gen )
def create_fea(dt):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
dt[lag_col] = dt[["id","sales"]].groupby("id")["sales"].shift(lag)
 wins = [7, 28]
for win in wins :
for lag,lag_col in zip(lags, lag_cols):
dt[f"rmean_{lag}_{win}"] = dt[["id", lag_col]].groupby("id")[lag_col].transform(lambda x : x.rolling(win ).mean())
 date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt["date"].dt, date_feat_func ).astype("int16")
<drop_column>
test_predicted=oh_to_class(test_predicted_oh )
df = reduce_mem_usage(df )<correct_missing_values>
sample_testing_images = next(test_data_gen)
df.dropna(inplace = True)
 df.shape<drop_column>
result_test = output_labels_to_map_labels(test_predicted,train_data_gen.class_indices, car_map_dict )
df = reduce_mem_usage(df )<prepare_x_and_y>
file_names = []
for name in test_data_gen.filenames:
file_names+=[name[6:]]
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(useless_cols)]
X_train = df[train_cols]
y_train = df["sales"]<create_dataframe>
result_image_prediction = np.concatenate(( file_names, result_test.astype(np.int32)) , axis = 1 )
train_data = lgb.Dataset(X_train, label = y_train, categorical_feature=cat_feats, free_raw_data=False)
 fake_valid_inds = np.random.choice(len(X_train), 1000000)
 fake_valid_data = lgb.Dataset(X_train.iloc[fake_valid_inds], label = y_train.iloc[fake_valid_inds],categorical_feature=cat_feats,
free_raw_data=False )<init_hyperparams>
result_df = pd.DataFrame(result_image_prediction, columns=['image', 'predictions'] )
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 1500,
}<train_model>
result_df.set_index('image' )
<drop_column><EOS> 
result_df.to_csv('submissionCommit8AugmentedDataE360.csv', index=False )