Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<prepare_x_and_y>
%matplotlib inline
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 100)
 print(os.listdir(".. /input"))
print(sys.version_info)
 np.random.seed(0)
 random.seed(0)
 RANDOM_SEED = 0
x_test = scipy.sparse.hstack([
test_desc_counts,
test_title_counts,
test.loc[:, predictors]
], format='csr')
 if VALID == True:
train_index, valid_index = train_test_split(np.arange(len(train)) , test_size=0.1, random_state=42)
 x_train = scipy.sparse.hstack([
train_desc_counts[train_index],
train_title_counts[train_index],
train.loc[train_index, predictors]
], format='csr')
 y_train = train.loc[train_index, target]
x_valid = scipy.sparse.hstack([
train_desc_counts[valid_index],
train_title_counts[valid_index],
train.loc[valid_index, predictors]
], format='csr')
 y_valid = train.loc[valid_index, target]
lgtrain = lgb.Dataset(x_train, y_train,
feature_name=list(feature_names),
categorical_feature = categorical)
 lgvalid = lgb.Dataset(x_valid, y_valid,
feature_name=list(feature_names),
categorical_feature = categorical)
 lgb_clf = lgb.train(
 lgbm_params,
lgtrain,
num_boost_round=20000,
valid_sets=[lgtrain, lgvalid],
valid_names=['train','valid'],
early_stopping_rounds=50,
verbose_eval=100)
 print("Model Evaluation Stage")
 del x_valid ; x_train; gc.collect()
else:
X = scipy.sparse.hstack([
train_desc_counts,
train_title_counts,
train.loc[: , predictors]
], format='csr')
 y = train.deal_probability
lgtrain = lgb.Dataset(X, y.values,
feature_name=list(feature_names),
categorical_feature = categorical)
 lgb_clf = lgb.train(
 lgbm_params,
lgtrain,
num_boost_round=2200)
 del X; gc.collect()
del train,test
gc.collect()<save_to_csv>
@jit
def rmse(predictions, targets):
return np.sqrt(((predictions - targets)** 2 ).mean())
 def trainer(X,
X_test,
y,
params,
folds,
columns=None,
eval_metric='mae' 
):
result_dict = {}
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
metrics_dict = {'mae': {'lgb_metric_name': 'mae',
'catboost_metric_name': 'MAE',
'sklearn_scoring_function': metrics.mean_absolute_error},
'group_mae': {'lgb_metric_name': 'mae',
'catboost_metric_name': 'MAE'
},
'mse': {'lgb_metric_name': 'mse',
'catboost_metric_name': 'MSE',
'sklearn_scoring_function': metrics.mean_squared_error}
}
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 model = lgbm.LGBMRegressor(**params, n_jobs = -1)
 if type(X)== np.ndarray:
X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
else:
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=0, early_stopping_rounds=300)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 if eval_metric != 'group_mae':
scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))
else:
scores.append(metrics_dict[eval_metric]['scoring_function'](y_valid, y_pred_valid, X_valid['type']))
prediction += y_pred
prediction /= folds.n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
return result_dict
def print_info(dataset, name):
s = "
" + "-"*70 + '
'
print(f"{s}",f"Info about {name} with shape {dataset.shape}",
f"{s}", dataset.head() ,
f"{s}", dataset.nunique() ,
f"{s}", dataset.describe().astype('int32'),
f"{s}",'Count NaN values
', dataset.isna().sum() , 
)
 def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024 ** 2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024 ** 2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(
 start_mem - end_mem)/ start_mem))
return df
subm = pd.read_csv('.. /input/avito-demand-prediction/sample_submission.csv')
 subm['deal_probability'] = np.clip(lgb_clf.predict(x_test), 0, 1)
 subm.to_csv('submission.csv', index=False )<load_from_csv>
nrows = None
items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 cats = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 try:
train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train_v2.csv', nrows=nrows)
 except:
train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv', nrows=nrows)
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID' )
print(os.listdir(".. /input"))
df = pd.read_csv(".. /input/avito-demand-prediction/train.csv")
 test = pd.read_csv(".. /input/avito-demand-prediction/test.csv")
 df_sorted = df.sort_values(by = ["image"])
 test_sorted = test.sort_values(by = ["image"] )<load_from_csv>
print(f"{len(set(train.shop_id)- set(test.shop_id)) } shops only in train.")
 print(f"{len(train.item_cnt_day[train.item_cnt_day < 0])} negative items_cnt_day")
 print(f"We need to predict clipped into [0, 20] range item_cnt_month column" )
tr_1 = pd.read_csv(".. /input/train-data-image-blurrness/1.csv")
 tr_2 = pd.read_csv(".. /input/train-data-image-blurrness/2.csv")
 tr_3 = pd.read_csv(".. /input/train-data-image-blurrness/3.csv")
 tr_4 = pd.read_csv(".. /input/train-data-image-blurrness/4.csv")
 tr_5 = pd.read_csv(".. /input/train-data-image-blurrness/5.csv")
 tr_6 = pd.read_csv(".. /input/train-data-image-blurrness/6.csv")
 tr_7 = pd.read_csv(".. /input/train-data-image-blurrness/7.csv")
 tr_8 = pd.read_csv(".. /input/train-data-image-blurrness/8.csv")
 tr_9 = pd.read_csv(".. /input/train-data-image-blurrness/9.csv")
 tr_10 = pd.read_csv(".. /input/train-data-image-blurrness/10.csv")
 tr_11 = pd.read_csv(".. /input/train-data-image-blurrness/11(12_13.5 ).csv")
 tr_12 = pd.read_csv(".. /input/train-data-image-blurrness/last.csv")
 frames = [tr_1, tr_2, tr_3, tr_4, tr_5, tr_6, tr_7, tr_8, tr_9, tr_10, tr_11, tr_12]
new = pd.concat(frames)
 new["File"] = new["File"].apply(lambda x : x.split("/")[-1].split(".")[0])
 new = new.sort_values(by = ["File"])
 scores = list(new["Score"].values)+ [-1] *(len(df)-len(new))
df_sorted["image_blurrness_score"] = scores
df = df_sorted.sort_index()<load_from_csv>
train = train[train.item_price<48000]
train = train[train.item_cnt_day<1000]
te_1 = pd.read_csv(".. /input/image-blurrness-test/test_1.csv")
 te_2 = pd.read_csv(".. /input/image-blurrness-test/test_2.csv")
 te_3 = pd.read_csv(".. /input/image-blurrness-test/test_3.csv")
 te_4 = pd.read_csv(".. /input/image-blurrness-test/test_4.csv")
 te_5 = pd.read_csv(".. /input/image-blurrness-test/test_5.csv")
 frames_te = [te_1, te_2, te_3, te_4, te_5]
new_te = pd.concat(frames_te)
 new_te["File"] = new_te["File"].apply(lambda x : x.split("/")[-1].split(".")[0])
 new_te = new_te.sort_values(by = ["File"])
 scores_te = list(new_te["Score"].values)+ [-1] *(len(test)-len(new_te))
test_sorted["image_blurrness_score"] = scores_te
test = test_sorted.sort_index()<drop_column>
cats.loc[32, "item_category_name"] = 'Карты оплаты -(Кино, Музыка, Игры)'
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
cats
df_train= df
df_test = test<define_variables>
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
parent_category_name_map = {"Личные вещи" : "Personal belongings",
"Для дома и дачи" : "For the home and garden",
"Бытовая электроника" : "Consumer electronics",
"Недвижимость" : "Real estate",
"Хобби и отдых" : "Hobbies & leisure",
"Транспорт" : "Transport",
"Услуги" : "Services",
"Животные" : "Animals",
"Для бизнеса" : "For business"} 
<define_variables>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
region_map = {"Свердловская область" : "Sverdlovsk oblast",
"Самарская область" : "Samara oblast",
"Ростовская область" : "Rostov oblast",
"Татарстан" : "Tatarstan",
"Волгоградская область" : "Volgograd oblast",
"Нижегородская область" : "Nizhny Novgorod oblast",
"Пермский край" : "Perm Krai",
"Оренбургская область" : "Orenburg oblast",
"Ханты-Мансийский АО" : "Khanty-Mansi Autonomous Okrug",
"Тюменская область" : "Tyumen oblast",
"Башкортостан" : "Bashkortostan",
"Краснодарский край" : "Krasnodar Krai",
"Новосибирская область" : "Novosibirsk oblast",
"Омская область" : "Omsk oblast",
"Белгородская область" : "Belgorod oblast",
"Челябинская область" : "Chelyabinsk oblast",
"Воронежская область" : "Voronezh oblast",
"Кемеровская область" : "Kemerovo oblast",
"Саратовская область" : "Saratov oblast",
"Владимирская область" : "Vladimir oblast",
"Калининградская область" : "Kaliningrad oblast",
"Красноярский край" : "Krasnoyarsk Krai",
"Ярославская область" : "Yaroslavl oblast",
"Удмуртия" : "Udmurtia",
"Алтайский край" : "Altai Krai",
"Иркутская область" : "Irkutsk oblast",
"Ставропольский край" : "Stavropol Krai",
"Тульская область" : "Tula oblast"} 
<define_variables>
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i],
sales.shop_id.unique() ,
sales.item_id.unique())) ,
dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True )
category_map = {"Одежда, обувь, аксессуары":"Clothing, shoes, accessories",
"Детская одежда и обувь":"Children's clothing and shoes",
"Товары для детей и игрушки":"Children's products and toys",
"Квартиры":"Apartments",
"Телефоны":"Phones",
"Мебель и интерьер":"Furniture and interior",
"Предложение услуг":"Offer services",
"Автомобили":"Cars",
"Ремонт и строительство":"Repair and construction",
"Бытовая техника":"Appliances",
"Товары для компьютера":"Products for computer",
"Дома, дачи, коттеджи":"Houses, villas, cottages",
"Красота и здоровье":"Health and beauty",
"Аудио и видео":"Audio and video",
"Спорт и отдых":"Sports and recreation",
"Коллекционирование":"Collecting",
"Оборудование для бизнеса":"Equipment for business",
"Земельные участки":"Land",
"Часы и украшения":"Watches and jewelry",
"Книги и журналы":"Books and magazines",
"Собаки":"Dogs",
"Игры, приставки и программы":"Games, consoles and software",
"Другие животные":"Other animals",
"Велосипеды":"Bikes",
"Ноутбуки":"Laptops",
"Кошки":"Cats",
"Грузовики и спецтехника":"Trucks and buses",
"Посуда и товары для кухни":"Tableware and goods for kitchen",
"Растения":"Plants",
"Планшеты и электронные книги":"Tablets and e-books",
"Товары для животных":"Pet products",
"Комнаты":"Room",
"Фототехника":"Photo",
"Коммерческая недвижимость":"Commercial property",
"Гаражи и машиноместа":"Garages and Parking spaces",
"Музыкальные инструменты":"Musical instruments",
"Оргтехника и расходники":"Office equipment and consumables",
"Птицы":"Birds",
"Продукты питания":"Food",
"Мотоциклы и мототехника":"Motorcycles and bikes",
"Настольные компьютеры":"Desktop computers",
"Аквариум":"Aquarium",
"Охота и рыбалка":"Hunting and fishing",
"Билеты и путешествия":"Tickets and travel",
"Водный транспорт":"Water transport",
"Готовый бизнес":"Ready business",
"Недвижимость за рубежом":"Property abroad"} 
<define_variables>
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
params_top35_map = {'Женская одежда':"Women's clothing",
'Для девочек':'For girls',
'Для мальчиков':'For boys',
'Продам':'Selling',
'С пробегом':'With mileage',
'Аксессуары':'Accessories',
'Мужская одежда':"Men's Clothing",
'Другое':'Other','Игрушки':'Toys',
'Детские коляски':'Baby carriages',
'Сдам':'Rent',
'Ремонт, строительство':'Repair, construction',
'Стройматериалы':'Building materials',
'iPhone':'iPhone',
'Кровати, диваны и кресла':'Beds, sofas and armchairs',
'Инструменты':'Instruments',
'Для кухни':'For kitchen',
'Комплектующие':'Accessories',
'Детская мебель':"Children's furniture",
'Шкафы и комоды':'Cabinets and chests of drawers',
'Приборы и аксессуары':'Devices and accessories',
'Для дома':'For home',
'Транспорт, перевозки':'Transport, transportation',
'Товары для кормления':'Feeding products',
'Samsung':'Samsung',
'Сниму':'Hire',
'Книги':'Books',
'Телевизоры и проекторы':'Televisions and projectors',
'Велосипеды и самокаты':'Bicycles and scooters',
'Предметы интерьера, искусство':'Interior items, art',
'Другая':'Other','Косметика':'Cosmetics',
'Постельные принадлежности':'Bed dress',
'С/х животные' :'Farm animals','Столы и стулья':'Tables and chairs'} 
<feature_engineering>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16)
 matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True )
df_train['region_en'] = df_train['region'].apply(lambda x : region_map[x])
 df_train['parent_category_name_en'] = df_train['parent_category_name'].apply(lambda x : parent_category_name_map[x])
 df_train['category_name_en'] = df_train['category_name'].apply(lambda x : category_map[x])
 del df_train['region']
del df_train['parent_category_name']
del df_train['category_name']
df_train
df_test['region_en'] = df_test['region'].apply(lambda x : region_map[x])
 df_test['parent_category_name_en'] = df_test['parent_category_name'].apply(lambda x : parent_category_name_map[x])
 df_test['category_name_en'] = df_test['category_name'].apply(lambda x : category_map[x])
 del df_test['region']
del df_test['parent_category_name']
del df_test['category_name']
df_test
df_train<load_from_csv>
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 matrix = matrix.drop(['item_name'], axis=1 )
regional = pd.read_csv(".. /input/regionaldata/regional.csv", index_col = [0])
 rDense = regional["Density_of_region(km2)"]
rRural = regional["Rural_%"]
rTime_zone = regional["Time_zone"]
rPopulation = regional["Total_population"]
rUrban = regional["Urban%"]
reg_index = np.array([regional.index[i].lower() for i in range(len(regional)) ])
 rDense.index = reg_index
rRural.index = reg_index
rTime_zone.index = reg_index
rPopulation.index = reg_index
rUrban.index = reg_index
df_region = df_train["region_en"]
reg_dense = np.array([rDense[df_region[i].lower() ] for i in range(len(df_train)) ])
 reg_rural = np.array([rRural[df_region[i].lower() ] for i in range(len(df_train)) ])
 reg_Time_zone = np.array([rTime_zone[df_region[i].lower() ] for i in range(len(df_train)) ])
 reg_Population = np.array([rPopulation[df_region[i].lower() ] for i in range(len(df_train)) ])
 reg_Urban = np.array([rUrban[df_region[i].lower() ] for i in range(len(df_train)) ])
 df_train["reg_dense"] = reg_dense
df_train["rural"] = reg_rural
df_train["reg_Time_zone"] = reg_Time_zone
df_train["reg_Population"] = reg_Population
df_train["reg_Urban"] = reg_Urban
df_train
reg_dense = np.array([rDense[df_region[i].lower() ] for i in range(len(df_test)) ])
 reg_rural = np.array([rRural[df_region[i].lower() ] for i in range(len(df_test)) ])
 reg_Time_zone = np.array([rTime_zone[df_region[i].lower() ] for i in range(len(df_test)) ])
 reg_Population = np.array([rPopulation[df_region[i].lower() ] for i in range(len(df_test)) ])
 reg_Urban = np.array([rUrban[df_region[i].lower() ] for i in range(len(df_test)) ])
 df_test["reg_dense"] = reg_dense
df_test["rural"] = reg_rural
df_test["reg_Time_zone"] = reg_Time_zone
df_test["reg_Population"] = reg_Population
df_test["reg_Urban"] = reg_Urban
df_train<drop_column>
lookback_range = list(range(1, 7 + 1))
new_features = []
for diff in tqdm(lookback_range):
feature_name = 'prev_shopitem_sales_' + str(diff)
 mx2 = matrix.copy()
mx2.loc[:, 'date_block_num'] += diff
mx2.rename(columns={'item_cnt_month': feature_name}, inplace=True)
 matrix = matrix.merge(mx2[['shop_id', 'item_id', 'date_block_num', feature_name]], on = ['shop_id', 'item_id', 'date_block_num'], how = 'left')
 matrix[feature_name] = matrix[feature_name].fillna(0)
 new_features.append(feature_name)
 groups = matrix.groupby(by = ['item_id', 'date_block_num'])
 for diff in tqdm(lookback_range):
feature_name = 'prev_item_sales_' + str(diff)
 result = groups.agg({'item_cnt_month':'mean'})
 result = result.reset_index()
result.loc[:, 'date_block_num'] += diff
result.rename(columns={'item_cnt_month': feature_name}, inplace=True)
 matrix = matrix.merge(result, on = ['item_id', 'date_block_num'], how = 'left')
 matrix[feature_name] = matrix[feature_name].fillna(0)
 new_features.append(feature_name)
 groups = matrix.groupby(by = ['item_id', 'date_block_num'])
 for diff in tqdm(lookback_range):
feature_name = '_prev_item_sales_' + str(diff)
 result = groups.agg({'item_cnt_month':'mean'})
 result = result.reset_index()
result.loc[:, 'date_block_num'] += diff
result.rename(columns={'item_cnt_month': feature_name}, inplace=True)
 matrix = matrix.merge(result, on = ['item_id', 'date_block_num'], how = 'left')
 matrix[feature_name] = matrix[feature_name].fillna(0)
 new_features.append(feature_name )
new_df_train = df_train.copy()
new_df_test = df_test.copy()
del new_df_test["image"]
del new_df_train["image"]
del new_df_test["activation_date"]
del new_df_train["activation_date"] 
<categorify>
me_cols = ['shop_id', 'item_id', 'city_code', 'item_category_id', 'type_code', 'subtype_code']
for cl in me_cols:
me_col_name = "me_" + cl
matrix.loc[:, me_col_name] = matrix[cl].map(matrix[cl].value_counts().apply(lambda x: x/len(matrix[cl])) )
stopWords_en = stopwords.words('english')
 stopWords_ru = stopwords.words('russian')
 tr =new_df_train.copy()
te =new_df_test.copy()
tri=tr.shape[0]
y = tr.deal_probability.copy()
lb=LabelEncoder()
List_Var=['item_id', 'description']
def Concat_Text(df,Columns,Name):
df=df.copy()
df.loc[:,Columns].fillna(" ",inplace=True)
 df[Name]=df[Columns[0]].astype('str')
 for col in Columns[1:]:
df[Name]=df[Name]+' '+df[col].astype('str')
 return df
def Ratio_Words(df):
df=df.copy()
df['description']=df['description'].astype('str')
 df['num_words_description']=df['description'].apply(lambda x:len(x.split()))
Unique_Words=df['description'].apply(lambda x: len(set(x.split())))
 df['Ratio_Words_description']=Unique_Words/df['num_words_description']
return df
def Lenght_Columns(df,Columns):
df=df.copy()
Columns_Len=['len_'+s for s in Columns]
for col in Columns:
df[col]=df[col].astype('str')
 for x,y in zip(Columns,Columns_Len):
df[y]=df[x].apply(len)
 return df
tr_te=tr[tr.columns.difference(["deal_probability"])].append(te)\ 
.pipe(Concat_Text,['city','param_1'],'txt1')\ 
.pipe(Concat_Text,['title','description'],'txt2' ).pipe(Ratio_Words ).pipe(Lenght_Columns,['title','description','param_1'] ).assign(category_name_en=lambda x: pd.Categorical(x['category_name_en'] ).codes,
parent_category_name_en=lambda x:pd.Categorical(x['parent_category_name_en'] ).codes,
region_en=lambda x:pd.Categorical(x['region_en'] ).codes, reg_Time_zone=lambda x:pd.Categorical(x['reg_Time_zone'] ).codes,
user_type=lambda x:pd.Categorical(x['user_type'] ).codes, image_top_1=lambda x:pd.Categorical(x['image_top_1'] ).codes,
param_1=lambda x:lb.fit_transform(x['param_1'].fillna('-1' ).astype('str')) ,
param_2=lambda x:lb.fit_transform(x['param_2'].fillna('-1' ).astype('str')) , param_3=lambda x:lb.fit_transform(x['param_3'].fillna('-1' ).astype('str')) ,
user_id=lambda x:lb.fit_transform(x['user_id'].astype('str')) , reg_dense=lambda x: np.log1p(x['reg_dense'].fillna(0)) ,
city=lambda x:lb.fit_transform(x['city'].astype('str')) , rural=lambda x: np.log1p(x['rural'].fillna(0)) ,
reg_Urban=lambda x: np.log1p(x['reg_Urban'].fillna(0)) ,
price=lambda x: np.log1p(x['price'].fillna(0)) , reg_Population=lambda x: np.log1p(x['reg_Population'].fillna(0)) ,
image_blurrness_score=lambda x: np.log1p(x['image_blurrness_score'].fillna(0)) ,
title=lambda x: x['title'].astype('str')).drop(labels=List_Var,axis=1)
 tr_te.price.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te.reg_Population.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te.reg_Urban.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te.reg_dense.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te.rural.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te.image_blurrness_score.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te
del tr,te
gc.collect()
tr_te.loc[:,'txt2']=tr_te.txt2.apply(lambda x:x.lower().replace("[^[:alpha:]]"," " ).replace("\\s+", " "))
print("Processing Text")
 print("Text 1")
 vec1=CountVectorizer(ngram_range=(1,2),dtype=np.uint8,min_df=5, binary=True,max_features=3000)
 m_tfidf1=vec1.fit_transform(tr_te.txt1)
 tr_te.drop(labels=['txt1'],inplace=True,axis=1)
 print("Text 2")
 vec2=TfidfVectorizer(ngram_range=(1,2),stop_words=stopWords_ru,min_df=3,max_df=0.4,sublinear_tf=True,norm='l2',max_features=5500,dtype=np.uint8)
 m_tfidf2=vec2.fit_transform(tr_te.txt2)
 tr_te.drop(labels=['txt2'],inplace=True,axis=1)
 print("Title")
 vec3=CountVectorizer(ngram_range=(3,6),analyzer='char_wb',dtype=np.uint8,min_df=5, binary=True,max_features=2000)
 m_tfidf3=vec3.fit_transform(tr_te.title)
 tr_te.drop(labels=['title'],inplace=True,axis=1)
 data = hstack(( tr_te.values,m_tfidf1,m_tfidf2,m_tfidf3)).tocsr()
print(data.shape)
 del tr_te,m_tfidf1,m_tfidf2,m_tfidf3
gc.collect()
dtest=data[tri:]
X=data[:tri]
del data
gc.collect()
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.15, random_state=23)
<create_dataframe>
del items, shops, cats, train
matrix = reduce_mem_usage(matrix)
 test = reduce_mem_usage(test )
dtrain =lgb.Dataset(data = X_train, label = y_train)
 dval =lgb.Dataset(data = X_valid, label = y_valid)
 Dparam = {'objective' : 'regression',
'boosting_type': 'gbdt',
'metric' : 'rmse',
'nthread' : 4,
'shrinkage_rate':0.02,
'max_depth':18,
'min_child_weight': 8,
'bagging_fraction':0.75,
'feature_fraction':0.75,
'lambda_l1':0,
'lambda_l2':0,
'num_leaves':31,
'verbosity' : -1}
print("Training Model")
 m_lgb=lgb.train(params=Dparam,train_set=dtrain,num_boost_round=40000, early_stopping_rounds=500, valid_sets=[dtrain,dval], verbose_eval=50,valid_names=['train','valid'])
 fig, ax = plt.subplots(figsize=(10, 14))
lgb.plot_importance(m_lgb, max_num_features=50, ax=ax)
 plt.title("Light GBM Feature Importance")
 Pred=m_lgb.predict(dtest)
 Pred[Pred<0]=0
Pred[Pred>1]=1
print("Output Model")
 LGB_text=pd.read_csv(".. /input/avito-demand-prediction/sample_submission.csv")
 LGB_text['deal_probability']=Pred
LGB_text.to_csv("no_Actdate.csv", index=False )<import_modules>
X_train = matrix[matrix.date_block_num <= 33].drop(['item_cnt_month'], axis=1)
 y = matrix[matrix.date_block_num <= 33]['item_cnt_month']
X_test = matrix[matrix.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 del matrix
gc.collect()
import os
import time
import numpy as np
import pandas as pd
from seaborn import countplot,lineplot, barplot
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import KFold
from sklearn.metrics import accuracy_score, confusion_matrix
from fastai import *
from fastai.tabular import *
from fastai.basic_data import DataBunch
from tqdm import tqdm_notebook
from bayes_opt import BayesianOptimization
import lightgbm as lgb
import xgboost as xgb
import catboost as cb<load_from_csv>
x_stsc = StandardScaler().fit(pd.concat([X_train, X_test]))
X_train = x_stsc.transform(X_train)
 X_test = x_stsc.transform(X_test )
tr = pd.read_csv('.. /input/X_train.csv')
 te = pd.read_csv('.. /input/X_test.csv')
 target = pd.read_csv('.. /input/y_train.csv')
 ss = pd.read_csv('.. /input/sample_submission.csv' )<count_values>
cv = KFold(3, shuffle=True )
len(tr.measurement_number.value_counts() )<concatenate>
def linear_model(
 X,
X_test,
y,
model,
params,
folds,
columns=None):
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 result_dict = {}
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(rmse(y_valid, y_pred_valid))
prediction += y_pred
gc.collect()
prediction /= folds.n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
return result_dict
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z<feature_engineering>
linreg_best_params= {
'n_jobs': -1
}
ridge_best_params = {
'alpha': 0.7,
'solver': 'lsqr',
'random_state': RANDOM_SEED
}
lr_model = LinearRegression(**linreg_best_params)
 ridge_model = Ridge(**ridge_best_params)
 linreg_results = linear_model(X_train, X_test, y,
lr_model,
linreg_best_params,
cv)
 ridge_results = linear_model(X_train, X_test, y,
ridge_model,
linreg_best_params,
cv)
def fe_step0(actual):
actual['norm_quat'] =(actual['orientation_X']**2 + actual['orientation_Y']**2 + actual['orientation_Z']**2 + actual['orientation_W']**2)
 actual['mod_quat'] =(actual['norm_quat'])**0.5
actual['norm_X'] = actual['orientation_X'] / actual['mod_quat']
actual['norm_Y'] = actual['orientation_Y'] / actual['mod_quat']
actual['norm_Z'] = actual['orientation_Z'] / actual['mod_quat']
actual['norm_W'] = actual['orientation_W'] / actual['mod_quat']
return actual
def fe_step1(actual):

 x, y, z, w = actual['norm_X'].tolist() , actual['norm_Y'].tolist() , actual['norm_Z'].tolist() , actual['norm_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 actual['euler_x'] = nx
actual['euler_y'] = ny
actual['euler_z'] = nz
return actual<feature_engineering>
gc.collect()
%%time
tr = fe_step0(tr)
 te = fe_step0(te)
 tr = fe_step1(tr)
 te = fe_step1(te )<feature_engineering>
def random_forest_regressor(
 X,
X_test,
y,
params,
folds,
columns=None):
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 result_dict = {}
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
if type(X)== np.ndarray:
X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
else:
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
model = RandomForestRegressor(**params)
 model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(rmse(y_valid, y_pred_valid))
prediction += y_pred
gc.collect()
prediction /= folds.n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
gc.collect()
return result_dict
def fe(data):
df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 +
data['angular_velocity_Z'])** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 +
data['linear_acceleration_Z'])**0.5
data['totl_xyz'] =(data['orientation_X']**2 + data['orientation_Y']**2 +
data['orientation_Z'])**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
for col in data.columns:
if col in ['row_id','series_id','measurement_number', 'group_id']:
continue
if col in ['surface']:
df[col] = data
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df<merge>
rf_best_params= {
'bootstrap': True,
'max_depth': 40,
'max_features': 'auto',
'min_samples_leaf': 4,
'min_samples_split': 10,
'n_estimators': 4,
'n_jobs': -1,
'verbose': 2
}
rf_results = random_forest_regressor(X_train, X_test, y,
rf_best_params,
cv )
tr = tr.merge(target, on='series_id', how='inner')
 tr = tr.drop(['group_id', 'series_id'], axis=1)
 tr.shape, te.shape<categorify>
params= {
'num_leaves': 64,
"max_depth": 8,
'max_bin': 32,
"iterations": 200,
"data_random_seed":0,
'n_estimators': 1000
}
lgb_results = trainer(X_train, X_test, y, params, cv )
le = LabelEncoder()
tr['surface'] = le.fit_transform(tr['surface'] )<data_type_conversions>
oof_preds = np.array([ridge_results['oof'],
linreg_results['oof'],
rf_results['oof'],
lgb_results['oof']
] ).T
preds = np.array([ridge_results['prediction'],
linreg_results['prediction'],
rf_results['prediction'],
lgb_results['prediction']
] ).T
tr.fillna(0, inplace = True)
 te.fillna(0, inplace = True )<categorify>
lvl2_model = LinearRegression()
lvl2_model.fit(oof_preds, y)
 lvl2_model.score(oof_preds, y)
 lvl2_result = lvl2_model.predict(preds )
tr.replace(-np.inf, 0, inplace = True)
 tr.replace(np.inf, 0, inplace = True)
 te.replace(-np.inf, 0, inplace = True)
 te.replace(np.inf, 0, inplace = True )<drop_column>
submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": lvl2_result
})
 submission['item_cnt_month'] = submission['item_cnt_month'].clip(0, 20)
 submission.to_csv('submission.csv',index=False )
<define_variables><EOS>