Kaggle competition: Lyft Motion Prediction for Autonomous Vehicles
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'shop_city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'shop_city'], how='left')
 matrix.date_item_city_avg_item_cnt = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], ['date_item_city_avg_item_cnt'])
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<data_type_conversions>
import matplotlib.pyplot as plt
import tensorflow as tf
ts = time.time()
group = train.groupby(["item_id"] ).agg({"item_price": ["mean"]})
 group.columns = ["item_avg_item_price"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["item_id"], how = "left")
 matrix["item_avg_item_price"] = matrix.item_avg_item_price.astype(np.float16)
 group = train.groupby(["date_block_num","item_id"] ).agg({"item_price": ["mean"]})
 group.columns = ["date_item_avg_item_price"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["date_block_num","item_id"], how = "left")
 matrix["date_item_avg_item_price"] = matrix.date_item_avg_item_price.astype(np.float16)
 lags = [1, 2, 3]
matrix = lag_feature(matrix, lags, ["date_item_avg_item_price"])
 for i in lags:
matrix["delta_price_lag_" + str(i)] =(matrix["date_item_avg_item_price_lag_" + str(i)]- matrix["item_avg_item_price"])/ matrix["item_avg_item_price"]
def select_trends(row):
for i in lags:
if row["delta_price_lag_" + str(i)]:
return row["delta_price_lag_" + str(i)]
return 0
matrix["delta_price_lag"] = matrix.apply(select_trends, axis = 1)
 matrix["delta_price_lag"] = matrix.delta_price_lag.astype(np.float16)
 matrix["delta_price_lag"].fillna(0 ,inplace = True)
 features_to_drop = ["item_avg_item_price", "date_item_avg_item_price"]
for i in lags:
features_to_drop.append("date_item_avg_item_price_lag_" + str(i))
features_to_drop.append("delta_price_lag_" + str(i))
matrix.drop(features_to_drop, axis = 1, inplace = True)
 time.time() - ts<merge>
tf.__version__
ts = time.time()
group = train.groupby(["date_block_num","shop_id"] ).agg({"revenue": ["sum"] })
 group.columns = ["date_shop_revenue"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group , on = ["date_block_num", "shop_id"], how = "left")
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(["shop_id"] ).agg({ "date_block_num":["mean"] })
 group.columns = ["shop_avg_revenue"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["shop_id"], how = "left")
 matrix["shop_avg_revenue"] = matrix.shop_avg_revenue.astype(np.float32)
 matrix["delta_revenue"] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix["delta_revenue"] = matrix["delta_revenue"].astype(np.float32)
 matrix = lag_feature(matrix, [1], ["delta_revenue"])
 matrix["delta_revenue_lag_1"] = matrix["delta_revenue_lag_1"].astype(np.float32)
 matrix.drop(["date_shop_revenue", "shop_avg_revenue", "delta_revenue"] ,axis = 1, inplace = True)
 time.time() - ts<categorify>
dataset_training = pd.read_csv("/kaggle/input/sf-crime/train.csv.zip")
 dataset_test = pd.read_csv("/kaggle/input/sf-crime/test.csv.zip" )
matrix["month"] = matrix["date_block_num"] % 12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix["days"] = matrix["month"].map(days ).astype(np.int8 )<feature_engineering>
dataset_training_new = dataset_training.drop(['Category', 'Descript', 'Resolution'], axis=1)
 dataset_test_new = dataset_test.drop('Id', axis=1)
 dataset_training_new['Dates'] = pd.to_datetime(dataset_training_new['Dates'])
 dataset_test_new['Dates'] = pd.to_datetime(dataset_test_new['Dates'] )
ts = time.time()
matrix["item_shop_first_sale"] = matrix["date_block_num"] - matrix.groupby(["item_id","shop_id"])["date_block_num"].transform('min')
 matrix["item_first_sale"] = matrix["date_block_num"] - matrix.groupby(["item_id"])["date_block_num"].transform('min')
 time.time() - ts<feature_engineering>
dataset_training_new['Year'] = dataset_training_new['Dates'].dt.year
dataset_test_new['Year'] = dataset_test_new['Dates'].dt.year
dataset_training_new['Month'] = dataset_training_new['Dates'].dt.month
dataset_test_new['Month'] = dataset_test_new['Dates'].dt.month
dataset_training_new['Day'] = dataset_training_new['Dates'].dt.day
dataset_test_new['Day'] = dataset_test_new['Dates'].dt.day
dataset_training_new['Hour'] = dataset_training_new['Dates'].dt.hour
dataset_test_new['Hour'] = dataset_test_new['Dates'].dt.hour
dataset_training_new['Minute'] = dataset_training_new['Dates'].dt.minute
dataset_test_new['Minute'] = dataset_test_new['Dates'].dt.minute
dataset_training_new['WeekDay'] = dataset_training_new['Dates'].dt.weekday
dataset_test_new['WeekDay'] = dataset_test_new['Dates'].dt.weekday
dataset_training_new.drop(['Dates', 'DayOfWeek', 'Address'], axis=1, inplace=True)
 dataset_test_new.drop(['Dates', 'DayOfWeek', 'Address'], axis=1, inplace=True)
ts = time.time()
matrix = matrix[matrix["date_block_num"] > 3]
time.time() - ts<set_options>
ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder() , [0])], remainder='passthrough')
 X = np.array(ct.fit_transform(dataset_training_new))
X_submission = np.array(ct.transform(dataset_test_new))
rcParams['figure.figsize'] = 12, 4<drop_column>
y = dataset_training['Category'].values
ohe = OneHotEncoder(sparse=False)
 y = ohe.fit_transform(y.reshape(-1, 1))
y.shape
data = matrix.copy()
del matrix
gc.collect()<prepare_x_and_y>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 1 )
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<set_options>
sc = MinMaxScaler()
X_train = sc.fit_transform(X_train)
 X_test = sc.transform(X_test )
del data
gc.collect() ;<train_model>
ann = tf.keras.models.Sequential()
ann.add(tf.keras.layers.Dense(units=200, activation='relu'))
ann.add(tf.keras.layers.Dense(units=200, activation='relu'))
ann.add(tf.keras.layers.Dense(units=200, activation='relu'))
ann.add(tf.keras.layers.Dense(units=200, activation='relu'))
ann.add(tf.keras.layers.Dense(units=39, activation='softmax'))
ann.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
 ann.fit(X_train, y_train, batch_size=32, epochs=25, validation_data=(X_test, y_test))
ts = time.time()
model = XGBRegressor(
 max_depth=10,
n_estimators=1000,
min_child_weight=0.5,
colsample_bytree=0.8,
subsample=0.8,
eta=0.1,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 20)
 time.time() - ts<save_to_csv>
X_submission = sc.transform(X_submission)
 y_pred = ann.predict(X_submission)
 m = np.max(y_pred, axis=1 ).reshape(-1, 1)
 predicted = np.array(( y_pred == m), dtype='int32')
 col_names = ohe.categories_[0]
df_submission = pd.DataFrame()
for i, entry in enumerate(col_names):
df_submission[entry] = predicted[:,i]
df_submission['Id'] = dataset_test['Id']
df_submission.set_index('Id', inplace=True)
 df_submission
<set_options><EOS> 
df_submission.to_csv('Submission.csv' )