Kaggle competition: Avito Demand Prediction Challenge
<SOS> metric: RMSE Kaggle data source: avito-demand-prediction-challenge<merge>
sns.set()
%matplotlib inline
NFOLDS = 5
SEED = 42
train_adj = p_oof[p_oof['Weeks']>=0].groupby(['Patient'],as_index=False)['Weeks'].min()
train_adj = pd.merge(train_adj,p_oof,how='left',on=['Patient','Weeks'])
 train_adj = train_adj.groupby(['Patient','Weeks'],as_index=False)[['Percent','FVC']].mean()
train_adj.rename(columns={'Weeks': 'First_week'}, inplace=True)
 train_adj.head(3 )<merge>
train = pd.read_csv('.. /input/avito-demand-prediction/train.csv')
 test = pd.read_csv('.. /input/avito-demand-prediction/test.csv')
 gp = pd.read_csv('.. /input/aggregated-features-lightgbm/aggregated_features.csv')
 train = train.merge(gp, on='user_id', how='left')
 test = test.merge(gp, on='user_id', how='left')
 agg_cols = list(gp.columns)[1:]
del gp
gc.collect()
train.head()
p_oof = pd.merge(p_oof,train_adj[['Patient','First_week']],how='left',on=['Patient'])
 p_oof_week_0 = p_oof[p_oof['Weeks']==p_oof['First_week']][['Patient','FVC','prediction']].copy()
p_oof_week_0.reset_index(inplace=True, drop=True)
 p_oof_week_0['prediction_w0_delta'] = p_oof_week_0['FVC'] - p_oof_week_0['prediction']
p_oof_week_0.head(3 )<merge>
for df in [train, test]:
df['description'].fillna('unknowndesc', inplace=True)
 df['title'].fillna('unknowntitle', inplace=True)
 df['weekday'] = pd.to_datetime(df['activation_date'] ).dt.day
for col in ['description', 'title']:
df['num_words_' + col] = df[col].apply(lambda comment: len(comment.split()))
df['num_unique_words_' + col] = df[col].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique_title'] = df['num_unique_words_title'] / df['num_words_title'] * 100
df['words_vs_unique_description'] = df['num_unique_words_description'] / df['num_words_description'] * 100
df['city'] = df['region'] + '_' + df['city']
df['num_desc_punct'] = df['description'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
for col in agg_cols:
df[col].fillna(-1, inplace=True )
p_oof = pd.merge(p_oof,p_oof_week_0[['Patient','prediction_w0_delta']],how='left',on=['Patient'])
 p_oof['predition_adjusted'] = p_oof['prediction'] + p_oof['prediction_w0_delta']<compute_test_metric>
count_vectorizer_title = CountVectorizer(stop_words=stopwords.words('russian'), lowercase=True, min_df=2)
 title_counts = count_vectorizer_title.fit_transform(train['title'].append(test['title']))
train_title_counts = title_counts[:len(train)]
test_title_counts = title_counts[len(train):]
count_vectorizer_desc = TfidfVectorizer(stop_words=stopwords.words('russian'),
lowercase=True, ngram_range=(1, 2),
max_features=17000)
 desc_counts = count_vectorizer_desc.fit_transform(train['description'].append(test['description']))
train_desc_counts = desc_counts[:len(train)]
test_desc_counts = desc_counts[len(train):]
train_title_counts.shape, train_desc_counts.shape
temp = p_oof.copy()
temp['for_eval'] =(temp['Weeks'] != temp['First_week'])
 temp = temp[temp['for_eval']==True]
print('mean OOF lap_log:',laplace_log_likelihood(temp['FVC'], temp['predition_adjusted'], temp['confidence']))<save_to_csv>
target = 'deal_probability'
predictors = [
'num_desc_punct',
'words_vs_unique_description', 'num_unique_words_description', 'num_unique_words_title', 'num_words_description', 'num_words_title',
'avg_times_up_user', 'avg_days_up_user', 'n_user_items',
'price', 'item_seq_number'
]
categorical = [
'image_top_1', 'param_1', 'param_2', 'param_3',
'city', 'region', 'category_name', 'parent_category_name', 'user_type'
]
predictors = predictors + categorical
save_oof = False
if save_oof:
check_me = train.copy()
check_me['prediction'] = preds_df['mean']
check_me['confidence'] = conf_df['mean']
check_me.head(3)
 check_me.to_csv("oof_name_here.csv", index=False )<prepare_output>
for feature in categorical:
print(f'Transforming {feature}...')
 encoder = LabelEncoder()
train[feature].fillna('unknown',inplace=True)
 test[feature].fillna('unknown',inplace=True)
 encoder.fit(train[feature].append(test[feature] ).astype(str))
train[feature] = encoder.transform(train[feature].astype(str))
test[feature] = encoder.transform(test[feature].astype(str))
my_sub = submission.copy()
my_sub.reset_index(inplace=True, drop=True)
 my_sub = my_sub[['Patient','Weeks']]
my_sub['FVC'] = preds_submission_df['mean'].values
my_sub['Confidence'] = conf_submission_df['mean'].values
my_sub.head()<save_to_csv>
train["price"] = np.log(train["price"]+0.001)
 train["price"].fillna(-999,inplace=True)
 train["image_top_1"].fillna(-999,inplace=True)
 test["price"] = np.log(test["price"]+0.001)
 test["price"].fillna(-999,inplace=True)
 test["image_top_1"].fillna(-999,inplace=True)
make_submission = True
if make_submission:
my_sub['Weeks'] = my_sub['Weeks'].astype(str)
 my_sub['Patient_Week'] = my_sub['Patient'] + '_' + my_sub['Weeks']
my_sub = my_sub[['Patient_Week','FVC','Confidence']]
print(my_sub.shape)
 my_sub.head()
my_sub.to_csv("submission.csv", index=False )<install_modules>
rounds = 20000
early_stop_rounds = 50
lgbm_params = {
'objective' : 'regression',
'metric' : 'rmse',
'num_leaves' : 300,
'learning_rate' : 0.021,
'feature_fraction' : 0.6,
'bagging_fraction' :.8,
'verbosity' : -1
}
feature_names = np.hstack([
count_vectorizer_desc.get_feature_names() ,
count_vectorizer_title.get_feature_names() ,
predictors
])
 print('Number of features:', len(feature_names))
<import_modules>
VALID = False
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import pydicom
import os
import matplotlib.gridspec as gridspec
from tqdm import tqdm
import tensorflow as tf
from tensorflow.keras.models import Sequential, load_model, Model
from tensorflow.keras.layers import ZeroPadding2D,UpSampling2D,ThresholdedReLU,Conv2DTranspose, Cropping2D, DepthwiseConv2D,add, Activation, LeakyReLU, Dense, Conv2D, GlobalMaxPooling2D , MaxPooling2D, Flatten,Concatenate, Input, Dropout, BatchNormalization, GlobalAveragePooling2D, SeparableConv2D, AveragePooling2D
import cv2
from tensorflow.keras.optimizers import RMSprop, Adam, SGD
from tensorflow.keras.metrics import TruePositives, FalsePositives, TrueNegatives, FalseNegatives, AUC, BinaryAccuracy
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.activations import softsign
from keras.initializers import Constant
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import KFold,StratifiedKFold,StratifiedShuffleSplit
import random
import keras.backend as K
import gc
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler,StandardScaler
from sklearn.metrics import mean_squared_error
from sklearn.neighbors import KNeighborsRegressor,NearestNeighbors
from sklearn.linear_model import LinearRegression,Ridge, Lasso, LogisticRegression
from sklearn.ensemble import AdaBoostRegressor,GradientBoostingRegressor
from sklearn.linear_model import ElasticNet, BayesianRidge
from sklearn.feature_selection import SelectFromModel
from sklearn.preprocessing import OneHotEncoder 
<load_from_csv>
x_test = scipy.sparse.hstack([
test_desc_counts,
test_title_counts,
test.loc[:, predictors]
], format='csr')
 if VALID == True:
train_index, valid_index = train_test_split(np.arange(len(train)) , test_size=0.1, random_state=42)
 x_train = scipy.sparse.hstack([
train_desc_counts[train_index],
train_title_counts[train_index],
train.loc[train_index, predictors]
], format='csr')
 y_train = train.loc[train_index, target]
x_valid = scipy.sparse.hstack([
train_desc_counts[valid_index],
train_title_counts[valid_index],
train.loc[valid_index, predictors]
], format='csr')
 y_valid = train.loc[valid_index, target]
lgtrain = lgb.Dataset(x_train, y_train,
feature_name=list(feature_names),
categorical_feature = categorical)
 lgvalid = lgb.Dataset(x_valid, y_valid,
feature_name=list(feature_names),
categorical_feature = categorical)
 lgb_clf = lgb.train(
 lgbm_params,
lgtrain,
num_boost_round=20000,
valid_sets=[lgtrain, lgvalid],
valid_names=['train','valid'],
early_stopping_rounds=50,
verbose_eval=100)
 print("Model Evaluation Stage")
 del x_valid ; x_train; gc.collect()
else:
X = scipy.sparse.hstack([
train_desc_counts,
train_title_counts,
train.loc[: , predictors]
], format='csr')
 y = train.deal_probability
lgtrain = lgb.Dataset(X, y.values,
feature_name=list(feature_names),
categorical_feature = categorical)
 lgb_clf = lgb.train(
 lgbm_params,
lgtrain,
num_boost_round=2200)
 del X; gc.collect()
del train,test
gc.collect()
<feature_engineering><EOS> 
subm = pd.read_csv('.. /input/avito-demand-prediction/sample_submission.csv')
 subm['deal_probability'] = np.clip(lgb_clf.predict(x_test), 0, 1)
 subm.to_csv('submission.csv', index=False )