Kaggle competition: Categorical Feature Encoding Challenge II
start_real = datetime.now()
np.random.seed(123 )<compute_test_metric>
df_train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')
 df_test = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')
 df_sub = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/sample_submission.csv' )
def rmsle(Y, Y_pred):
assert Y.shape == Y_pred.shape
return np.sqrt(np.mean(np.square(Y_pred - Y)))
<load_from_csv>
df_train.sort_index(inplace=True )
%%time
train_df = pd.read_table('.. /input/train.tsv')
 test_df = pd.read_table('.. /input/test.tsv')
 print(train_df.shape, test_df.shape )<drop_column>
y_train = df_train['target']
test_id = df_test['id']
df_train.drop(['target', 'id'], axis=1, inplace=True)
 df_test.drop('id', axis=1, inplace=True )
train_df = train_df.drop(train_df[(train_df.price < 3.0)].index)
 train_df.shape<feature_engineering>
cat_feat_to_encode = df_train.columns.tolist()
smoothing=0.20
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state= 1024, shuffle=True ).split(df_train, y_train):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(df_train.iloc[tr_idx, :], y_train.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(df_train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(df_train, y_train)
 df_train = oof.sort_index()
df_test = ce_target_encoder.transform(df_test )
<feature_engineering>
x_train = df_train.iloc[:,:].values
x_test = df_test.iloc[:,:].values
%%time
def wordCount(text):
try:
if text == 'No description yet':
return 0
else:
text = text.lower()
words = [w for w in text.split(" ")]
return len(words)
 except:
return 0
train_df['desc_len'] = train_df['item_description'].apply(lambda x: wordCount(x))
test_df['desc_len'] = test_df['item_description'].apply(lambda x: wordCount(x))
train_df['name_len'] = train_df['name'].apply(lambda x: wordCount(x))
test_df['name_len'] = test_df['name'].apply(lambda x: wordCount(x))
train_df.head()<feature_engineering>
classifier = lgb.LGBMClassifier(**{
'learning_rate': 0.05,
'feature_fraction': 0.1,
'min_data_in_leaf' : 12,
'max_depth': 3,
'reg_alpha': 1,
'reg_lambda': 1,
'objective': 'binary',
'metric': 'auc',
'n_jobs': -1,
'n_estimators' : 5000,
'feature_fraction_seed': 42,
'bagging_seed': 42,
'boosting_type': 'gbdt',
'verbose': 1,
'is_unbalance': True,
'boost_from_average': False} )
%%time
def split_cat(text):
try: return text.split("/")
 except: return("No Label", "No Label", "No Label")
 train_df['subcat_0'], train_df['subcat_1'], train_df['subcat_2'] = \
zip(*train_df['category_name'].apply(lambda x: split_cat(x)))
 test_df['subcat_0'], test_df['subcat_1'], test_df['subcat_2'] = \
zip(*test_df['category_name'].apply(lambda x: split_cat(x)) )<feature_engineering>
classifier.fit(x_train, y_train )
full_set = pd.concat([train_df,test_df])
 all_brands = set(full_set['brand_name'].values)
 train_df.brand_name.fillna(value="missing", inplace=True)
 test_df.brand_name.fillna(value="missing", inplace=True)
 premissing = len(train_df.loc[train_df['brand_name'] == 'missing'])
 def brandfinder(line):
brand = line[0]
name = line[1]
namesplit = name.split(' ')
 if brand == 'missing':
for x in namesplit:
if x in all_brands:
return name
if name in all_brands:
return name
return brand
train_df['brand_name'] = train_df[['brand_name','name']].apply(brandfinder, axis = 1)
 test_df['brand_name'] = test_df[['brand_name','name']].apply(brandfinder, axis = 1)
 found = premissing-len(train_df.loc[train_df['brand_name'] == 'missing'])
 print(found )<train_model>
y_pred = classifier.predict_proba(x_test)[:,1]
<concatenate><EOS> 
df_sub = pd.DataFrame()
df_sub['id'] = test_id
df_sub['target'] = y_pred
df_sub.to_csv('submission.csv', index=False )