Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<categorify>
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
%%time
def fill_missing_values(df):
df.category_name.fillna(value="missing", inplace=True)
 df.brand_name.fillna(value="missing", inplace=True)
 df.item_description.fillna(value="missing", inplace=True)
 df.item_description.replace('No description yet',"missing", inplace=True)
 return df
print("Filling missing data...")
 full_df = fill_missing_values(full_df)
 print(full_df.category_name[1] )<categorify>
train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')
 sample_submission = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/sample_submission.csv' )
%%time
print("Processing categorical data...")
 le = LabelEncoder()
le.fit(full_df.category_name)
 full_df['category'] = le.transform(full_df.category_name)
 le.fit(full_df.brand_name)
 full_df.brand_name = le.transform(full_df.brand_name)
 le.fit(full_df.subcat_0)
 full_df.subcat_0 = le.transform(full_df.subcat_0)
 le.fit(full_df.subcat_1)
 full_df.subcat_1 = le.transform(full_df.subcat_1)
 le.fit(full_df.subcat_2)
 full_df.subcat_2 = le.transform(full_df.subcat_2)
 del le<feature_engineering>
all_df = pd.concat([train, test], axis=0, ignore_index=True )
<feature_engineering>
all_df.index.nunique() , len(all_df.index )
%%time
print("Transforming text data to sequences...")
 raw_text = np.hstack([full_df.item_description.str.lower() , full_df.name.str.lower() , full_df.category_name.str.lower() ])
 print(" Fitting tokenizer...")
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text)
 print(" Transforming text to sequences...")
 full_df['seq_item_description'] = tok_raw.texts_to_sequences(full_df.item_description.str.lower())
 full_df['seq_name'] = tok_raw.texts_to_sequences(full_df.name.str.lower())
 del tok_raw<define_variables>
all_df = all_df.drop('id', axis=1 )
MAX_NAME_SEQ = 10
MAX_ITEM_DESC_SEQ = 75
MAX_CATEGORY_SEQ = 8
MAX_TEXT = np.max([
np.max(full_df.seq_name.max()),
np.max(full_df.seq_item_description.max()),
])+ 100
MAX_CATEGORY = np.max(full_df.category.max())+ 1
MAX_BRAND = np.max(full_df.brand_name.max())+ 1
MAX_CONDITION = np.max(full_df.item_condition_id.max())+ 1
MAX_DESC_LEN = np.max(full_df.desc_len.max())+ 1
MAX_NAME_LEN = np.max(full_df.name_len.max())+ 1
MAX_SUBCAT_0 = np.max(full_df.subcat_0.max())+ 1
MAX_SUBCAT_1 = np.max(full_df.subcat_1.max())+ 1
MAX_SUBCAT_2 = np.max(full_df.subcat_2.max())+ 1<categorify>
nunique_vals = list()
for column in all_df:
nunique_vals.append(all_df[column].nunique())
 pd.DataFrame({'columns': all_df.columns,
'num_of_unique': nunique_vals} )
%%time
def get_rnn_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ),
'item_desc': pad_sequences(dataset.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ),
'brand_name': np.array(dataset.brand_name),
'category': np.array(dataset.category),
'item_condition': np.array(dataset.item_condition_id),
'num_vars': np.array(dataset[["shipping"]]),
'desc_len': np.array(dataset[["desc_len"]]),
'name_len': np.array(dataset[["name_len"]]),
'subcat_0': np.array(dataset.subcat_0),
'subcat_1': np.array(dataset.subcat_1),
'subcat_2': np.array(dataset.subcat_2),
}
return X
train = full_df[:n_trains]
dev = full_df[n_trains:n_trains+n_devs]
test = full_df[n_trains+n_devs:]
X_train = get_rnn_data(train)
 Y_train = train.target.values.reshape(-1, 1)
 X_dev = get_rnn_data(dev)
 Y_dev = dev.target.values.reshape(-1, 1)
 X_test = get_rnn_data(test )<compute_test_metric>
for column in all_df.columns:
unique_values = all_df[column].unique()
print(f'Statistics fot column: {column}')
 print(f'Column unique values:
{unique_values}')
 print(f'Number of unique values: {len(unique_values)}')
 print(f'Number of NAN values: {all_df[column].isna().sum() }')
 print('_' * 50 )
def root_mean_squared_logarithmic_error(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+ 1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+ 1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1)+0.0000001)
 def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)+0.0000001 )<define_variables>
month_temperature = all_df.groupby(['month', 'ord_2'])['ord_2'].count().to_frame()
month_temperature = month_temperature.rename(columns={'ord_2': 'num_of_days'})
 month_temperature = month_temperature.reset_index()
np.random.seed(123)
 def new_rnn_model(lr=0.001, decay=0.0):
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 desc_len = Input(shape=[1], name="desc_len")
 name_len = Input(shape=[1], name="name_len")
 subcat_0 = Input(shape=[1], name="subcat_0")
 subcat_1 = Input(shape=[1], name="subcat_1")
 subcat_2 = Input(shape=[1], name="subcat_2")
 emb_name = Embedding(MAX_TEXT, 20 )(name)
 emb_item_desc = Embedding(MAX_TEXT, 60 )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND, 10 )(brand_name)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 emb_desc_len = Embedding(MAX_DESC_LEN, 5 )(desc_len)
 emb_name_len = Embedding(MAX_NAME_LEN, 5 )(name_len)
 emb_subcat_0 = Embedding(MAX_SUBCAT_0, 10 )(subcat_0)
 emb_subcat_1 = Embedding(MAX_SUBCAT_1, 10 )(subcat_1)
 emb_subcat_2 = Embedding(MAX_SUBCAT_2, 10 )(subcat_2)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_item_condition)
 , Flatten()(emb_desc_len)
 , Flatten()(emb_name_len)
 , Flatten()(emb_subcat_0)
 , Flatten()(emb_subcat_1)
 , Flatten()(emb_subcat_2)
 , rnn_layer1
, rnn_layer2
, num_vars
])
 main_l = Dropout(0.1 )(Dense(512,kernel_initializer='normal',activation='relu' )(main_l))
main_l = Dropout(0.1 )(Dense(256,kernel_initializer='normal',activation='relu' )(main_l))
main_l = Dropout(0.1 )(Dense(128,kernel_initializer='normal',activation='relu' )(main_l))
main_l = Dropout(0.1 )(Dense(64,kernel_initializer='normal',activation='relu' )(main_l))
output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name , item_condition,
num_vars, desc_len, name_len, subcat_0, subcat_1, subcat_2], output)
 optimizer = Adam(lr=lr, decay=decay)
 model.compile(loss = 'mse', optimizer = optimizer)
 return model
model = new_rnn_model()
model.summary()
del model<train_model>
all_df['ord_5_1'] = all_df['ord_5'].str[0]
all_df['ord_5_2'] = all_df['ord_5'].str[1]
all_df = all_df.drop('ord_5', axis=1 )
%%time
BATCH_SIZE = 512 * 3
epochs = 2
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
steps = int(len(X_train['name'])/ BATCH_SIZE)* epochs
lr_init, lr_fin = 0.005, 0.001
lr_decay = exp_decay(lr_init, lr_fin, steps)
 rnn_model = new_rnn_model(lr=lr_init, decay=lr_decay)
 rnn_model.fit(
 X_train, Y_train, epochs=epochs, batch_size=BATCH_SIZE,
validation_data=(X_dev, Y_dev), verbose=1,
)<predict_on_test>
all_df['nan_features'] = all_df.isna().sum(axis=1 )
%%time
print("Evaluating the model on validation data...")
 Y_dev_preds_rnn = rnn_model.predict(X_dev, batch_size=BATCH_SIZE)
 print(" RMSLE error:", rmsle(Y_dev, Y_dev_preds_rnn))<predict_on_test>
all_df['month_sin'] = np.sin(( all_df['month'] - 1)*(2.0 * np.pi / 12))
all_df['month_cos'] = np.cos(( all_df['month'] - 1)*(2.0 * np.pi / 12))
all_df['day_sin'] = np.sin(( all_df['day'] - 1)*(2.0 * np.pi / 7))
all_df['day_cos'] = np.cos(( all_df['day'] - 1)*(2.0 * np.pi / 7))
rnn_preds = rnn_model.predict(X_test, batch_size=BATCH_SIZE, verbose=1)
 rnn_preds = np.expm1(rnn_preds )<concatenate>
categorical = ['bin_0', 'bin_1', 'bin_2', 'bin_3',
'bin_4', 'day', 'month', 'nom_0',
'nom_1', 'nom_2', 'nom_3', 'nom_4',
'nom_5', 'nom_6', 'nom_7', 'nom_8',
'nom_9', 'ord_0', 'ord_1', 'ord_2',
'ord_3', 'ord_4', 'ord_5_1', 'ord_5_2']
nom_5_9 = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
continuous = ['month_sin', 'month_cos',
'day_sin', 'day_cos']
full_df = pd.concat([train_df, dev_df, test_df] )<data_type_conversions>
features = [x for x in all_df.columns
if x not in ['id', 'target'] + continuous]
%%time
print("Handling missing values...")
 full_df['category_name'] = full_df['category_name'].fillna('missing' ).astype(str)
 full_df['subcat_0'] = full_df['subcat_0'].astype(str)
 full_df['subcat_1'] = full_df['subcat_1'].astype(str)
 full_df['subcat_2'] = full_df['subcat_2'].astype(str)
 full_df['brand_name'] = full_df['brand_name'].fillna('missing' ).astype(str)
 full_df['shipping'] = full_df['shipping'].astype(str)
 full_df['item_condition_id'] = full_df['item_condition_id'].astype(str)
 full_df['desc_len'] = full_df['desc_len'].astype(str)
 full_df['name_len'] = full_df['name_len'].astype(str)
 full_df['item_description'] = full_df['item_description'].fillna('No description yet' ).astype(str )<feature_engineering>
for feat in tqdm(features):
lbl_enc = preprocessing.LabelEncoder()
all_df[feat] = lbl_enc.fit_transform(all_df[feat].\
fillna('-1' ).\
astype(str ).values)
 all_df['target'] = all_df['target'].fillna(-1)
 all_df[continuous] = all_df[continuous].fillna(-2 )
%%time
print("Vectorizing data...")
 default_preprocessor = CountVectorizer().build_preprocessor()
def build_preprocessor(field):
field_idx = list(full_df.columns ).index(field)
 return lambda x: default_preprocessor(x[field_idx])
 vectorizer = FeatureUnion([ 
('name', CountVectorizer(
 ngram_range=(1, 2),
max_features=50000,
preprocessor=build_preprocessor('name'))), 
('subcat_0', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('subcat_0'))), 
('subcat_1', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('subcat_1'))), 
('subcat_2', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('subcat_2'))), 
('brand_name', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('brand_name'))), 
('shipping', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('shipping'))), 
('item_condition_id', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('item_condition_id'))), 
('desc_len', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('desc_len'))), 
('name_len', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('name_len'))), 
('item_description', TfidfVectorizer(
 ngram_range=(1, 3),
max_features=100000,
preprocessor=build_preprocessor('item_description'))),
])
 X = vectorizer.fit_transform(full_df.values)
 X_train = X[:n_trains]
Y_train = train_df.target.values.reshape(-1, 1)
 X_dev = X[n_trains:n_trains+n_devs]
Y_dev = dev_df.target.values.reshape(-1, 1)
 X_test = X[n_trains+n_devs:]
print(X.shape, X_train.shape, X_dev.shape, X_test.shape )<train_on_grid>
def cramers_v(x, y):

 confusion_matrix = pd.crosstab(x, y)
 chi2 = chi2_contingency(confusion_matrix)[0]
n = confusion_matrix.sum().sum()
r, k = confusion_matrix.shape
phi_2 = chi2 / n
phi2corr = max(0, phi_2 -(( k - 1)*(r - 1)) /(n - 1))
rcorr = r -(( r - 1)** 2)/(n - 1)
 kcorr = k -(( k - 1)** 2)/(n - 1)
 res = np.sqrt(phi2corr / min(kcorr - 1, rcorr - 1))
return res
def conditional_entropy(x, y):

 y_counter = Counter(y)
 xy_counter = Counter(( list(zip(x, y))))
total_occurrences = sum(y_counter.values())
 entropy = 0.0
for xy in xy_counter.keys() :
p_xy = xy_counter[xy] / total_occurrences
p_y = y_counter[xy[1]] / total_occurrences
entropy += p_xy * np.log(p_y / p_xy)
 return entropy
def theils_u(x, y):

 s_xy = conditional_entropy(x, y)
 x_counter = Counter(x)
 total_occurrences = sum(x_counter.values())
 p_x = list(map(lambda n: n / total_occurrences, x_counter.values()))
s_x = entropy(p_x)
 if s_x == 0:
return 1
else:
return(s_x - s_xy)/ s_x
%%time
print("Fitting Ridge model on training examples...")
 ridge_model = Ridge(
 solver='auto', fit_intercept=True, alpha=1.0,
max_iter=100, normalize=False, tol=0.05, random_state = 1, 
)
 ridge_modelCV = RidgeCV(
 fit_intercept=True, alphas=[5.0],
normalize=False, cv = 2, scoring='neg_mean_squared_error', 
)
 ridge_model.fit(X_train, Y_train)
 ridge_modelCV.fit(X_train, Y_train )<predict_on_test>
plt.rcParams.update({'font.size': 10} )
Y_dev_preds_ridge = ridge_model.predict(X_dev)
 Y_dev_preds_ridge = Y_dev_preds_ridge.reshape(-1, 1)
 print("RMSL error on dev set:", rmsle(Y_dev, Y_dev_preds_ridge))<compute_train_metric>
to_dummies = ['day', 'month', 'nom_0',
'nom_1', 'nom_2', 'nom_3', 'nom_4']
Y_dev_preds_ridgeCV = ridge_modelCV.predict(X_dev)
 Y_dev_preds_ridgeCV = Y_dev_preds_ridgeCV.reshape(-1, 1)
 print("CV RMSL error on dev set:", rmsle(Y_dev, Y_dev_preds_ridgeCV))<predict_on_test>
%%time
all_df = pd.get_dummies(all_df,
columns=to_dummies,
sparse=True,
dtype=np.int8 )
%%time
ridge_preds = ridge_model.predict(X_test)
 ridge_preds = np.expm1(ridge_preds)
 ridgeCV_preds = ridge_modelCV.predict(X_test)
 ridgeCV_preds = np.expm1(ridgeCV_preds )<compute_train_metric>
all_df.isna().sum().sum()
%%time
def aggregate_predicts3(Y1, Y2, Y3, ratio1, ratio2):
assert Y1.shape == Y2.shape
return Y1 * ratio1 + Y2 * ratio2 + Y3 *(1.0 - ratio1-ratio2)
<compute_train_metric>
%%time
train = all_df[:train.shape[0]]
test = all_df[train.shape[0]:]
%%time
best1 = 0
best2 = 0
lowest = 0.99
for i in range(100):
for j in range(100):
r = i*0.01
r2 = j*0.01
if r+r2 < 1.0:
Y_dev_preds = aggregate_predicts3(Y_dev_preds_rnn, Y_dev_preds_ridgeCV, Y_dev_preds_ridge, r, r2)
 fpred = rmsle(Y_dev, Y_dev_preds)
 if fpred < lowest:
best1 = r
best2 = r2
lowest = fpred
Y_dev_preds = aggregate_predicts3(Y_dev_preds_rnn, Y_dev_preds_ridgeCV, Y_dev_preds_ridge, best1, best2 )<compute_test_metric>
train.isna().sum().sum() , test.isna().sum().sum()
print(best1)
 print(best2)
 print("(Best)RMSL error for RNN + Ridge + RidgeCV on dev set:", rmsle(Y_dev, Y_dev_preds))<save_to_csv>
%%time
train_data = train.drop('target', axis=1 ).to_numpy()
train_target = train['target'].to_numpy()
test_data = test.drop('target', axis=1 ).to_numpy()
preds = aggregate_predicts3(rnn_preds, ridgeCV_preds, ridge_preds, best1, best2)
 submission = pd.DataFrame({
"test_id": test_df.test_id,
"price": preds.reshape(-1),
})
 submission.to_csv("./rnn_ridge_submission_best.csv", index=False )<save_to_csv>
%%time
categorical = all_df.drop(['target'] + continuous,
axis=1 ).columns
cat_cols_idx, cont_cols_idx = list() , list()
for idx, column in enumerate(all_df.drop('target',
axis=1 ).columns):
if column in categorical:
cat_cols_idx.append(idx)
 elif column in continuous:
cont_cols_idx.append(idx )
preds = aggregate_predicts3(rnn_preds, ridgeCV_preds, ridge_preds, 0.334, 0.333)
 submission = pd.DataFrame({
"test_id": test_df.test_id,
"price": preds.reshape(-1),
})
 submission.to_csv("./rnn_ridge_submission_mean.csv", index=False )<save_to_csv>
train_data, train_target, test_data
preds = aggregate_predicts3(rnn_preds, ridgeCV_preds, ridge_preds, best1-0.1, best2+0.1)
 submission = pd.DataFrame({
"test_id": test_df.test_id,
"price": preds.reshape(-1),
})
 submission.to_csv("./rnn_ridge_submission_var_1.csv", index=False )<save_to_csv>
import torch
from torch.utils.data import Dataset, DataLoader
from torch.utils.data.sampler import SubsetRandomSampler
preds = aggregate_predicts3(rnn_preds, ridgeCV_preds, ridge_preds, best1+0.1, best2-0.1)
 submission = pd.DataFrame({
"test_id": test_df.test_id,
"price": preds.reshape(-1),
})
 submission.to_csv("./rnn_ridge_submission_var_2.csv", index=False )<define_variables>
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministick = True
torch.backends.cudnn.benchmark = False
set_seed(27 )
stop_real = datetime.now()
execution_time_real = stop_real-start_real
print(execution_time_real )<categorify>
class ClassificationDataset(Dataset):
def __init__(self, data, targets=None,
is_train=True, cat_cols_idx=None,
cont_cols_idx=None):
self.data = data
self.targets = targets
self.is_train = is_train
self.cat_cols_idx = cat_cols_idx
self.cont_cols_idx = cont_cols_idx
def __getitem__(self, idx):
row = self.data[idx].astype('float32')
 data_cat = []
data_cont = []
result = None
if self.cat_cols_idx:
data_cat = torch.tensor(row[self.cat_cols_idx])
 if self.cont_cols_idx:
data_cont = torch.tensor(row[self.cont_cols_idx])
 data = [data_cat, data_cont]
if self.is_train:
result = {'data': data,
'target': torch.tensor(self.targets[idx])}
else:
result = {'data': data}
return result
def __len__(self):
return(len(self.data))
def transform_values(data):
data.fillna(value='missing', inplace = True)
 return data<load_from_csv>
train_dataset = ClassificationDataset(train_data,
targets=train_target,
cat_cols_idx=cat_cols_idx,
cont_cols_idx=cont_cols_idx)
 test_dataset = ClassificationDataset(test_data,
cat_cols_idx=cat_cols_idx,
cont_cols_idx=cont_cols_idx,
is_train=False )
df_train = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_test = pd.read_csv('.. /input/test.tsv', sep='\t')
 df_train = transform_values(df_train)
 df_test = transform_values(df_test )<feature_engineering>
def split_dataset(trainset, valid_size=0.2, batch_size=64):
num_train = len(trainset)
 indices = list(range(num_train))
np.random.shuffle(indices)
 split = int(np.floor(valid_size * num_train))
valid_idx, train_idx = indices[:split], indices[split:]
valid_sampler = SubsetRandomSampler(valid_idx)
 train_sampler = SubsetRandomSampler(train_idx)
 valid_loader = DataLoader(trainset,
batch_size=batch_size,
sampler=valid_sampler)
 train_loader = DataLoader(trainset,
batch_size=batch_size,
sampler=train_sampler)
 return train_loader, valid_loader
def transform_category_name(category_name):
try:
main, sub1, sub2= category_name.split('/')
 return main, sub1, sub2
except:
return 'missing', 'missing', 'missing'
df_train['category_main'], df_train['category_sub1'], df_train['category_sub2'] = zip(*df_train['category_name'].apply(transform_category_name))
df_train[df_train.price != 0.0]
df_train[df_train.item_description != 'missing']
df_test['category_main'], df_test['category_sub1'], df_test['category_sub2'] = zip(*df_test['category_name'].apply(transform_category_name))
df_test[df_test.item_description != 'missing']<categorify>
train_loader, valid_loader = split_dataset(train_dataset,
valid_size=0.2,
batch_size=2000 )
cv = CountVectorizer(ngram_range=(1,2), min_df=7, stop_words='english')
 X_name_train = cv.fit_transform(df_train['name'])
 X_name_test = cv.transform(df_test['name'])
<feature_engineering>
class ClassificationEmbdNN(torch.nn.Module):
def __init__(self, emb_dims, no_of_cont=None):
super(ClassificationEmbdNN, self ).__init__()
self.emb_layers = torch.nn.ModuleList([torch.nn.Embedding(x, y)
 for x, y in emb_dims])
 no_of_embs = sum([y for x, y in emb_dims])
 self.no_of_embs = no_of_embs
self.emb_dropout = torch.nn.Dropout(0.2)
 self.no_of_cont = 0
if no_of_cont:
self.no_of_cont = no_of_cont
self.bn_cont = torch.nn.BatchNorm1d(no_of_cont)
 self.fc1 = torch.nn.Linear(in_features=self.no_of_embs + self.no_of_cont,
out_features=256)
 self.dropout1 = torch.nn.Dropout(0.2)
 self.bn1 = torch.nn.BatchNorm1d(256)
 self.act1 = torch.nn.ReLU()
self.fc2 = torch.nn.Linear(in_features=256,
out_features=256)
 self.dropout2 = torch.nn.Dropout(0.2)
 self.bn2 = torch.nn.BatchNorm1d(256)
 self.act2 = torch.nn.ReLU()
self.fc3 = torch.nn.Linear(in_features=256,
out_features=64)
 self.dropout3 = torch.nn.Dropout(0.2)
 self.bn3 = torch.nn.BatchNorm1d(64)
 self.act3 = torch.nn.ReLU()
self.fc4 = torch.nn.Linear(in_features=64,
out_features=1)
 self.act4 = torch.nn.Sigmoid()
def forward(self, x_cat, x_cont=None):
if self.no_of_embs != 0:
x = [emb_layer(x_cat[:, i])
 for i, emb_layer in enumerate(self.emb_layers)]
x = torch.cat(x, 1)
 x = self.emb_dropout(x)
 if self.no_of_cont != 0:
x_cont = self.bn_cont(x_cont)
 if self.no_of_embs != 0:
x = torch.cat([x, x_cont], 1)
 else:
x = x_cont
x = self.fc1(x)
 x = self.dropout1(x)
 x = self.bn1(x)
 x = self.act1(x)
 x = self.fc2(x)
 x = self.dropout2(x)
 x = self.bn2(x)
 x = self.act2(x)
 x = self.fc3(x)
 x = self.dropout3(x)
 x = self.bn3(x)
 x = self.act3(x)
 x = self.fc4(x)
 x = self.act4(x)
 return x
cat_cv1 = CountVectorizer()
cat_cv2 = CountVectorizer()
cat_cv3 = CountVectorizer()
X_catmain_train = cat_cv1.fit_transform(df_train['category_main'])
 X_catsub1_train = cat_cv2.fit_transform(df_train['category_sub1'])
 X_catsub2_train = cat_cv3.fit_transform(df_train['category_sub2'])
 X_catmain_test = cat_cv1.transform(df_test['category_main'])
 X_catsub1_test = cat_cv2.transform(df_test['category_sub1'])
 X_catsub2_test = cat_cv3.transform(df_test['category_sub2'] )<categorify>
def train_network(model, train_loader, valid_loader,
loss_func, optimizer, n_epochs=20,
saved_model='model.pt'):
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 model.to(device)
 train_losses = list()
valid_losses = list()
valid_loss_min = np.Inf
for epoch in range(n_epochs):
train_loss = 0.0
valid_loss = 0.0
train_auc = 0.0
valid_auc = 0.0
model.train()
for batch in tqdm(train_loader):
optimizer.zero_grad()
output = model(batch['data'][0].to(device,
dtype=torch.long),
batch['data'][1].to(device,
dtype=torch.float))
loss = loss_func(output, batch['target'].to(device,
dtype=torch.float))
loss.backward()
optimizer.step()
train_auc += metrics.roc_auc_score(batch['target'].cpu().numpy() ,
output.detach().cpu().numpy())
 train_loss += loss.item() * batch['data'][0].size(0)
 model.eval()
for batch in tqdm(valid_loader):
output = model(batch['data'][0].to(device,
dtype=torch.long),
batch['data'][1].to(device,
dtype=torch.float))
loss = loss_func(output, batch['target'].to(device,
dtype=torch.float))
valid_auc += metrics.roc_auc_score(batch['target'].cpu().numpy() ,
output.detach().cpu().numpy())
 valid_loss += loss.item() * batch['data'][0].size(0)
 train_loss = np.sqrt(train_loss / len(train_loader.sampler.indices))
valid_loss = np.sqrt(valid_loss / len(valid_loader.sampler.indices))
train_auc = train_auc / len(train_loader)
 valid_auc = valid_auc / len(valid_loader)
 train_losses.append(train_loss)
 valid_losses.append(valid_loss)
 print('Epoch: {}.Training loss: {:.6f}.Validation loss: {:.6f}' 
.format(epoch, train_loss, valid_loss))
print('Training AUC: {:.6f}.Validation AUC: {:.6f}' 
.format(train_auc, valid_auc))
if valid_loss < valid_loss_min:
print('Validation loss decreased({:.6f} --> {:.6f} ).Saving model...' 
.format(valid_loss_min, valid_loss))
torch.save(model.state_dict() , saved_model)
 valid_loss_min = valid_loss
return train_losses, valid_losses
vectorizer = TfidfVectorizer(ngram_range=(1,2), max_features=90000, strip_accents="ascii",
stop_words='english')
 X_description_train = vectorizer.fit_transform(df_train['item_description'])
 X_description_test = vectorizer.transform(df_test['item_description'] )<categorify>
cat_dim = [int(all_df[col].nunique())for col in categorical]
cat_dim = [[x, min(200,(x + 1)// 2)] for x in cat_dim]
for el in cat_dim:
if el[0] < 10:
el[1] = el[0]
cat_dim
lb = LabelBinarizer()
X_brand_train = lb.fit_transform(df_train['brand_name'])
 X_brand_test = lb.transform(df_test['brand_name'] )<categorify>
model = ClassificationEmbdNN(emb_dims=cat_dim,
no_of_cont=len(continuous))
loss_func = torch.nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters() , lr=0.1)
 train_losses, valid_losses = train_network(model=model,
train_loader=train_loader,
valid_loader=valid_loader,
loss_func=loss_func,
optimizer=optimizer,
n_epochs=3,
saved_model='simple_nn.pt' )
X_shipping_train = pd.get_dummies(df_train['shipping'])
 X_condition_train = pd.get_dummies(df_train['item_condition_id'])
 X_shipping_test = pd.get_dummies(df_test['shipping'])
 X_condition_test = pd.get_dummies(df_test['item_condition_id'] )<concatenate>
def predict(data_loader, model):
model.eval()
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 model.to(device)
 with torch.no_grad() :
predictions = None
for i, batch in enumerate(tqdm(data_loader)) :
output = model(batch['data'][0].to(device,
dtype=torch.long),
batch['data'][1].to(device,
dtype=torch.float)).cpu().numpy()
if i == 0:
predictions = output
else:
predictions = np.vstack(( predictions, output))
return predictions
vectorized_data_train = hstack(( X_name_train, X_catmain_train, X_catsub1_train, X_catsub2_train,
X_description_train, X_shipping_train, X_condition_train))
vectorized_data_test = hstack(( X_name_test, X_catmain_test, X_catsub1_test, X_catsub2_test,
X_description_test, X_shipping_test, X_condition_test)) 
<train_model>
model.load_state_dict(torch.load('simple_nn.pt'))
test_loader = DataLoader(test_dataset,
batch_size=1000)
 nn_predictions = predict(test_loader, model )
Y_train = np.log1p(df_train['price'])
 clf = Ridge(alpha=5.3, fit_intercept=True, normalize=False,
copy_X=True, max_iter=None, tol=0.01, solver='auto', random_state=None)
 clf.fit(vectorized_data_train, Y_train)
 Y_pred_test = clf.predict(vectorized_data_test )<save_to_csv>
nn_predictions_df = pd.DataFrame({'id': sample_submission['id'], 'target': nn_predictions.squeeze() } )
submission = pd.DataFrame(data=df_test[['test_id']])
 submission['price'] = np.expm1(Y_pred_test)
 submission.to_csv("submission_ridge.csv", index=False )<compute_test_metric>
from catboost import CatBoostClassifier
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5 
<set_options>
X = train.drop('target', axis=1)
 y = train['target']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=27)
 test_data = test.drop('target', axis=1 )
%matplotlib inline
sns.set()
plt.rcParams["figure.figsize"] =(20, 10)
 pd.options.display.float_format = '{:.2f}'.format<load_from_csv>
cat_features = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4', 'nom_5', 'nom_6', 'nom_7',
'nom_8', 'nom_9', 'ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4',
'ord_5_1', 'ord_5_2', 'nan_features', 'day_0', 'day_1', 'day_2',
'day_3', 'day_4', 'day_5', 'day_6', 'day_7', 'month_0', 'month_1',
'month_2', 'month_3', 'month_4', 'month_5', 'month_6', 'month_7',
'month_8', 'month_9', 'month_10', 'month_11', 'month_12', 'nom_0_0',
'nom_0_1', 'nom_0_2', 'nom_0_3', 'nom_1_0', 'nom_1_1', 'nom_1_2',
'nom_1_3', 'nom_1_4', 'nom_1_5', 'nom_1_6', 'nom_2_0', 'nom_2_1',
'nom_2_2', 'nom_2_3','nom_2_4', 'nom_2_5', 'nom_2_6', 'nom_3_0',
'nom_3_1', 'nom_3_2','nom_3_3', 'nom_3_4', 'nom_3_5', 'nom_3_6',
'nom_4_0', 'nom_4_1', 'nom_4_2', 'nom_4_3', 'nom_4_4']
df = pd.read_csv('.. /input/train.tsv', sep='\t', index_col=0 )<load_from_csv>
best_params = {
'bagging_temperature': 0.8,
'depth': 5,
'iterations': 1000,
'l2_leaf_reg': 30,
'learning_rate': 0.05,
'random_strength': 0.8
}
model_cat = CatBoostClassifier(**best_params,
loss_function='Logloss',
eval_metric='AUC',
nan_mode='Min',
thread_count=4,
task_type='GPU',
verbose=True)
 model_cat.fit(X_train, y_train,
eval_set=(X_test, y_test),
cat_features=cat_features,
verbose_eval=300,
early_stopping_rounds=500,
use_best_model=True,
plot=False)
df_test = pd.read_csv('.. /input/test.tsv', sep='\t', index_col=0 )<count_missing_values>
cat_predictions = model_cat.predict_proba(test_data)[:, 1]
sum(df.name.isnull() )<count_unique_values>
cat_predictions_df = pd.DataFrame({'id': sample_submission['id'],
'target': cat_predictions} )
df.category_name.unique().shape<count_missing_values>
res_sub = pd.DataFrame({'id': sample_submission['id']})
 res_sub.head()
sum(df.category_name.isnull() )<sort_values>
res_sub['target'] = round(( cat_predictions_df['target'] + nn_predictions_df['target'])/ 2, 2 )
df.groupby("category_name" ).agg({"price": "mean"} ).sort_values("price", ascending=False ).head(10 )<count_missing_values>
res_sub.to_csv('res_sub.csv', index=False )