Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<compute_train_metric>
%matplotlib inline
plt.style.use('ggplot')
 py.init_notebook_mode(connected=True)
 warnings.filterwarnings("ignore")
 pd.set_option('max_colwidth', 500)
 pd.set_option('max_columns', 500)
 pd.set_option('max_rows', 100)
 def kappa(y_true, y_pred):
return cohen_kappa_score(y_true, y_pred, weights='quadratic' )
model = LogisticRegression(C=3,dual = True)
 NBLR_score=[]
for clas in class_names:
print(clas)
 y = train[clas].values
r = np.log(pr(1,y)/ pr(0,y))
x_nb = train_matrix.multiply(r)
 score = cross_val_score(model,x_nb,y,scoring='accuracy',cv=5)
 NBLR_score.append(score.mean() )<prepare_output>
breeds = pd.read_csv('.. /input/breed_labels.csv')
 colors = pd.read_csv('.. /input/color_labels.csv')
 states = pd.read_csv('.. /input/state_labels.csv')
 train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv')
 sub = pd.read_csv('.. /input/test/sample_submission.csv')
 train['dataset_type'] = 'train'
test['dataset_type'] = 'test'
all_data = pd.concat([train, test] )
DF_score = pd.DataFrame(index=class_names)
 DF_score['NB'] = NB_score
DF_score['LR'] = LR_score
DF_score['NBLR'] = NBLR_score
DF_score<predict_on_test>
ax.patches
preds = np.zeros(( len(test), len(class_names)))
 for i, j in enumerate(class_names):
print('fit', j)
 m,r = get_mdl(train[j])
 preds[:,i] = m.predict_proba(test_matrix.multiply(r)) [:,1]<save_to_csv>
print('Most popular pet names and AdoptionSpeed')
 for n in train['Name'].value_counts().index[:5]:
print(n)
 print(train.loc[train['Name'] == n, 'AdoptionSpeed'].value_counts().sort_index())
 print('' )
submid = pd.DataFrame({'id': test["id"]})
 submission = pd.concat([submid, pd.DataFrame(preds, columns = class_names)], axis=1)
 submission.to_csv('submission_1.csv', index=False )<import_modules>
train['Name'] = train['Name'].fillna('Unnamed')
 test['Name'] = test['Name'].fillna('Unnamed')
 all_data['Name'] = all_data['Name'].fillna('Unnamed')
 train['No_name'] = 0
train.loc[train['Name'] == 'Unnamed', 'No_name'] = 1
test['No_name'] = 0
test.loc[test['Name'] == 'Unnamed', 'No_name'] = 1
all_data['No_name'] = 0
all_data.loc[all_data['Name'] == 'Unnamed', 'No_name'] = 1
print(f"Rate of unnamed pets in train data: {train['No_name'].sum() * 100 / train['No_name'].shape[0]:.4f}%.")
 print(f"Rate of unnamed pets in test data: {test['No_name'].sum() * 100 / test['No_name'].shape[0]:.4f}%." )
print(check_output(["ls", ".. /input"] ).decode("utf8")) 
<load_from_csv>
all_data[all_data['Name'].apply(lambda x: len(str(x)))== 3]['Name'].value_counts().tail()
train_df = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/train.csv")
 test_df = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/test.csv")
<prepare_x_and_y>
train['Age'].value_counts().head(10 )
train_df['comment_text'].fillna("unknown", inplace=True)
 test_df['comment_text'].fillna("unknown", inplace=True)
 train_text_data = train_df['comment_text']
test_text_data = test_df['comment_text']
train_y = train_df[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].values
con_data = pd.concat([train_text_data,test_text_data])
<define_variables>
train['Pure_breed'] = 0
train.loc[train['Breed2'] == 0, 'Pure_breed'] = 1
test['Pure_breed'] = 0
test.loc[test['Breed2'] == 0, 'Pure_breed'] = 1
all_data['Pure_breed'] = 0
all_data.loc[all_data['Breed2'] == 0, 'Pure_breed'] = 1
print(f"Rate of pure breed pets in train data: {train['Pure_breed'].sum() * 100 / train['Pure_breed'].shape[0]:.4f}%.")
 print(f"Rate of pure breed pets in test data: {test['Pure_breed'].sum() * 100 / test['Pure_breed'].shape[0]:.4f}%." )
max_features=100000
maxlen=150
embed_size=300<string_transform>
breeds_dict = {k: v for k, v in zip(breeds['BreedID'], breeds['BreedName'])}
tokenizer = text.Tokenizer(num_words=max_features, lower=True)
<train_model>
train['Breed1_name'] = train['Breed1'].apply(lambda x: '_'.join(breeds_dict[x].split())if x in breeds_dict else 'Unknown')
 train['Breed2_name'] = train['Breed2'].apply(lambda x: '_'.join(breeds_dict[x])if x in breeds_dict else '-')
 test['Breed1_name'] = test['Breed1'].apply(lambda x: '_'.join(breeds_dict[x].split())if x in breeds_dict else 'Unknown')
 test['Breed2_name'] = test['Breed2'].apply(lambda x: '_'.join(breeds_dict[x].split())if x in breeds_dict else '-')
 all_data['Breed1_name'] = all_data['Breed1'].apply(lambda x: '_'.join(breeds_dict[x].split())if x in breeds_dict else 'Unknown')
 all_data['Breed2_name'] = all_data['Breed2'].apply(lambda x: '_'.join(breeds_dict[x].split())if x in breeds_dict else '-' )
tokenizer.fit_on_texts(list(con_data)) 
<string_transform>
( all_data['Breed1_name'] + '__' + all_data['Breed2_name'] ).value_counts().head(15 )
train_x = tokenizer.texts_to_sequences(train_text_data)
 test_x = tokenizer.texts_to_sequences(test_text_data)
<concatenate>
colors_dict = {k: v for k, v in zip(colors['ColorID'], colors['ColorName'])}
train['Color1_name'] = train['Color1'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 train['Color2_name'] = train['Color2'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 train['Color3_name'] = train['Color3'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 test['Color1_name'] = test['Color1'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 test['Color2_name'] = test['Color2'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 test['Color3_name'] = test['Color3'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 all_data['Color1_name'] = all_data['Color1'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 all_data['Color2_name'] = all_data['Color2'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 all_data['Color3_name'] = all_data['Color3'].apply(lambda x: colors_dict[x] if x in colors_dict else '' )
train_x_pad = sequence.pad_sequences(train_x, maxlen=maxlen)
 test_x_pad = sequence.pad_sequences(test_x, maxlen=maxlen )<string_transform>
gender_dict = {1: 'Male', 2: 'Female', 3: 'Mixed'}
for i in all_data['Type'].unique() :
for j in all_data['Gender'].unique() :
df = all_data.loc[(all_data['Type'] == i)&(all_data['Gender'] == j)]
top_colors = list(df['full_color'].value_counts().index)[:5]
j = gender_dict[j]
print(f"Most popular colors of {j} {i}s: {' '.join(top_colors)}" )
word_index = tokenizer.word_index<define_variables>
images = [i.split('-')[0] for i in os.listdir('.. /input/train_images/')]
size_dict = {1: 'Small', 2: 'Medium', 3: 'Large', 4: 'Extra Large'}
for t in all_data['Type'].unique() :
for m in all_data['MaturitySize'].unique() :
df = all_data.loc[(all_data['Type'] == t)&(all_data['MaturitySize'] == m)]
top_breeds = list(df['Breed1_name'].value_counts().index)[:5]
m = size_dict[m]
print(f"Most common Breeds of {m} {t}s:")
 fig = plt.figure(figsize=(25, 4))
for i, breed in enumerate(top_breeds):
b_df = df.loc[(df['Breed1_name'] == breed)&(df['PetID'].isin(images)) , 'PetID']
if len(b_df)> 1:
pet_id = b_df.values[1]
else:
pet_id = b_df.values[0]
ax = fig.add_subplot(1, 5, i+1, xticks=[], yticks=[])
 im = Image.open(".. /input/train_images/" + pet_id + '-1.jpg')
 plt.imshow(im)
 ax.set_title(f'Breed: {breed}')
 plt.show() ;
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'<categorify>
c = 0
strange_pets = []
for i, row in all_data[all_data['Breed1_name'].str.contains('air')].iterrows() :
if 'Short' in row['Breed1_name'] and row['FurLength'] == 1:
pass
elif 'Medium' in row['Breed1_name'] and row['FurLength'] == 2:
pass
elif 'Long' in row['Breed1_name'] and row['FurLength'] == 3:
pass
else:
c += 1
strange_pets.append(( row['PetID'], row['Breed1_name'], row['FurLength']))
print(f"There are {c} pets whose breed and fur length don't match" )
embeddings_index = {}
with open(EMBEDDING_FILE, encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs<feature_engineering>
train['health'] = train['Vaccinated'].astype(str)+ '_' + train['Dewormed'].astype(str)+ '_' + train['Sterilized'].astype(str)+ '_' + train['Health'].astype(str)
 test['health'] = test['Vaccinated'].astype(str)+ '_' + test['Dewormed'].astype(str)+ '_' + test['Sterilized'].astype(str)+ '_' + test['Health'].astype(str)
 make_factor_plot(df=train.loc[train['health'].isin(list(train.health.value_counts().index[:5])) ], x='health', col='AdoptionSpeed', title='Counts of pets by main health conditions and Adoption Speed' )
word_index = tokenizer.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector 
<compute_train_metric>
train['Quantity'].value_counts().head(10 )
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))<choose_model_class>
all_data.sort_values('Fee', ascending=False)[['Name', 'Description', 'Fee', 'AdoptionSpeed', 'dataset_type']].head(10 )
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.10 )(x)
 x = Bidirectional(LSTM(128, return_sequences=True, dropout=0.10, recurrent_dropout=0.10))(x)
 x = Conv1D(64, kernel_size=3, padding='valid', kernel_initializer='glorot_uniform' )(x)
 x = Conv1D(64, kernel_size=3, padding='valid', kernel_initializer='glorot_uniform' )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 out = Dense(6, activation='sigmoid' )(x)
 model = Model(inp, out)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<split>
states_dict = {k: v for k, v in zip(states['StateID'], states['StateName'])}
train['State_name'] = train['State'].apply(lambda x: '_'.join(states_dict[x].split())if x in states_dict else 'Unknown')
 test['State_name'] = test['State'].apply(lambda x: '_'.join(states_dict[x].split())if x in states_dict else 'Unknown')
 all_data['State_name'] = all_data['State'].apply(lambda x: '_'.join(states_dict[x].split())if x in states_dict else 'Unknown' )
batch_size = 128
epochs = 4
X_tra, X_val, y_tra, y_val = train_test_split(train_x_pad, train_y, train_size=0.9, random_state=233 )<train_model>
all_data['State_name'].value_counts(normalize=True ).head()
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),callbacks = callbacks_list,verbose=1)
 model.load_weights(filepath)
 print('Predicting.... ')
 predictions = model.predict(test_x_pad,batch_size=1024,verbose=1)
<save_to_csv>
all_data['RescuerID'].value_counts().head()
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']] = predictions
submission.to_csv('submission.csv', index=False )<load_from_csv>
train['VideoAmt'].value_counts()
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
train = pd.read_csv('.. /input/train.csv' ).fillna(' ')
 test = pd.read_csv('.. /input/test.csv' ).fillna(' ' )<concatenate>
print(F'Maximum amount of photos in {train["PhotoAmt"].max() }')
 train['PhotoAmt'].value_counts().head()
train_text = train['comment_text']
test_text = test['comment_text']
all_text = pd.concat([train_text, test_text] )<feature_engineering>
tokenizer = TweetTokenizer()
vectorizer = TfidfVectorizer(ngram_range=(1, 2), tokenizer=tokenizer.tokenize)
 vectorizer.fit(all_data['Description'].fillna('' ).values)
 X_train = vectorizer.transform(train['Description'].fillna(''))
rf = RandomForestClassifier(n_estimators=20)
 rf.fit(X_train, train['AdoptionSpeed'] )
word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
stop_words='english',
ngram_range=(1,2),
max_features=20000)
 word_vectorizer.fit(all_text)
 print("fitted...")
 train_word_features = word_vectorizer.transform(train_text)
 print("train...")
 test_word_features = word_vectorizer.transform(test_text)
<feature_engineering>
train['Description'] = train['Description'].fillna('')
 test['Description'] = test['Description'].fillna('')
 all_data['Description'] = all_data['Description'].fillna('')
 train['desc_length'] = train['Description'].apply(lambda x: len(x))
train['desc_words'] = train['Description'].apply(lambda x: len(x.split()))
test['desc_length'] = test['Description'].apply(lambda x: len(x))
test['desc_words'] = test['Description'].apply(lambda x: len(x.split()))
all_data['desc_length'] = all_data['Description'].apply(lambda x: len(x))
all_data['desc_words'] = all_data['Description'].apply(lambda x: len(x.split()))
train['averate_word_length'] = train['desc_length'] / train['desc_words']
test['averate_word_length'] = test['desc_length'] / test['desc_words']
all_data['averate_word_length'] = all_data['desc_length'] / all_data['desc_words']
char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
stop_words='english',
ngram_range=(2,6),
max_features=50000)
 char_vectorizer.fit(all_text)
 print("fitted...")
 train_char_features = char_vectorizer.transform(train_text)
 print("train...")
 test_char_features = char_vectorizer.transform(test_text )<save_to_csv>
sentiment_dict = {}
for filename in os.listdir('.. /input/train_sentiment/'):
with open('.. /input/train_sentiment/' + filename, 'r')as f:
sentiment = json.load(f)
 pet_id = filename.split('.')[0]
sentiment_dict[pet_id] = {}
sentiment_dict[pet_id]['magnitude'] = sentiment['documentSentiment']['magnitude']
sentiment_dict[pet_id]['score'] = sentiment['documentSentiment']['score']
sentiment_dict[pet_id]['language'] = sentiment['language']
for filename in os.listdir('.. /input/test_sentiment/'):
with open('.. /input/test_sentiment/' + filename, 'r')as f:
sentiment = json.load(f)
 pet_id = filename.split('.')[0]
sentiment_dict[pet_id] = {}
sentiment_dict[pet_id]['magnitude'] = sentiment['documentSentiment']['magnitude']
sentiment_dict[pet_id]['score'] = sentiment['documentSentiment']['score']
sentiment_dict[pet_id]['language'] = sentiment['language']
train_features = hstack([train_char_features, train_word_features])
 test_features = hstack([test_char_features, test_word_features])
 scores = []
submission = pd.DataFrame.from_dict({'id': test['id']})
 for class_name in class_names:
print(class_name)
 train_target = train[class_name]
classifier = LogisticRegression()
classifier.fit(train_features, train_target)
 submission[class_name] = classifier.predict_proba(test_features)[:, 1]
submission.to_csv('submission.csv', index=False )<import_modules>
train['lang'] = train['PetID'].apply(lambda x: sentiment_dict[x]['language'] if x in sentiment_dict else 'no')
 train['magnitude'] = train['PetID'].apply(lambda x: sentiment_dict[x]['magnitude'] if x in sentiment_dict else 0)
 train['score'] = train['PetID'].apply(lambda x: sentiment_dict[x]['score'] if x in sentiment_dict else 0)
 test['lang'] = test['PetID'].apply(lambda x: sentiment_dict[x]['language'] if x in sentiment_dict else 'no')
 test['magnitude'] = test['PetID'].apply(lambda x: sentiment_dict[x]['magnitude'] if x in sentiment_dict else 0)
 test['score'] = test['PetID'].apply(lambda x: sentiment_dict[x]['score'] if x in sentiment_dict else 0)
 all_data['lang'] = all_data['PetID'].apply(lambda x: sentiment_dict[x]['language'] if x in sentiment_dict else 'no')
 all_data['magnitude'] = all_data['PetID'].apply(lambda x: sentiment_dict[x]['magnitude'] if x in sentiment_dict else 0)
 all_data['score'] = all_data['PetID'].apply(lambda x: sentiment_dict[x]['score'] if x in sentiment_dict else 0 )
<load_from_csv>
cols_to_use = ['Type', 'Age', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State', 'RescuerID', 'health', 'Free', 'score',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'No_name', 'Pure_breed', 'desc_length', 'desc_words', 'averate_word_length', 'magnitude']
train = train[[col for col in cols_to_use if col in train.columns]]
test = test[[col for col in cols_to_use if col in test.columns]]
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
train = pd.read_csv('.. /input/train.csv' ).fillna(' ')
 test = pd.read_csv('.. /input/test.csv' ).fillna(' ')
 train_text = train['comment_text']
test_text = test['comment_text']
all_text = pd.concat([train_text, test_text] )<feature_engineering>
cat_cols = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'State', 'RescuerID',
'No_name', 'Pure_breed', 'health', 'Free']
word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
stop_words='english',
ngram_range=(1, 1),
max_features=10000)
 word_vectorizer.fit(all_text)
 train_word_features = word_vectorizer.transform(train_text)
 test_word_features = word_vectorizer.transform(test_text )<feature_engineering>
more_cols = []
for col1 in cat_cols:
for col2 in cat_cols:
if col1 != col2 and col1 not in ['RescuerID', 'State'] and col2 not in ['RescuerID', 'State']:
train[col1 + '_' + col2] = train[col1].astype(str)+ '_' + train[col2].astype(str)
 test[col1 + '_' + col2] = test[col1].astype(str)+ '_' + test[col2].astype(str)
 more_cols.append(col1 + '_' + col2)
 cat_cols = cat_cols + more_cols
char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
stop_words='english',
ngram_range=(2, 6),
max_features=50000)
 char_vectorizer.fit(all_text)
 train_char_features = char_vectorizer.transform(train_text)
 test_char_features = char_vectorizer.transform(test_text )<create_dataframe>
%%time
indexer = {}
for col in cat_cols:
_, indexer[col] = pd.factorize(train[col].astype(str))
for col in tqdm_notebook(cat_cols):
train[col] = indexer[col].get_indexer(train[col].astype(str))
test[col] = indexer[col].get_indexer(test[col].astype(str))
train_features = hstack([train_char_features, train_word_features])
 test_features = hstack([test_char_features, test_word_features])
 scores = []
submission = pd.DataFrame.from_dict({'id': test['id']} )<train_model>
y = train['AdoptionSpeed']
train = train.drop(['AdoptionSpeed'], axis=1 )
for class_name in class_names:
train_target = train[class_name]
classifier = Ridge(alpha=20, copy_X=True, fit_intercept=True, solver='auto',
max_iter=100, normalize=False, random_state=0, tol=0.0025)
 classifier.fit(train_features, train_target)
 submission[class_name] = classifier.predict(test_features )<save_to_csv>
n_fold = 5
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=15 )
submission.to_csv('submission.csv', index=False )<save_to_csv>
def train_model(X=train, X_test=test, y=y, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, averaging='usual', make_oof=False):
result_dict = {}
if make_oof:
oof = np.zeros(( len(X), 5))
prediction = np.zeros(( len(X_test), 5))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y)) :
gc.collect()
print('Fold', fold_n + 1, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
train_data = lgb.Dataset(X_train, label=y_train, categorical_feature = cat_cols)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature = cat_cols)
 model = lgb.train(params,
train_data,
num_boost_round=20000,
valid_sets = [train_data, valid_data],
verbose_eval=500,
early_stopping_rounds = 200)
 del train_data, valid_data
y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 del X_valid
gc.collect()
y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test), ntree_limit=model.best_ntree_limit)
 if model_type == 'lcv':
model = LogisticRegressionCV(scoring='neg_log_loss', cv=3, multi_class='multinomial')
 model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 if model_type == 'cat':
model = CatBoostClassifier(iterations=20000, loss_function='MultiClass', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test ).reshape(-1,)
 if make_oof:
oof[valid_index] = y_pred_valid
scores.append(kappa(y_valid, y_pred_valid.argmax(1)))
 print('Fold kappa:', kappa(y_valid, y_pred_valid.argmax(1)))
 print('')
 if averaging == 'usual':
prediction += y_pred
elif averaging == 'rank':
prediction += pd.Series(y_pred ).rank().values
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importance()
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
if plot_feature_importance:
feature_importance["importance"] /= n_fold
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
result_dict['feature_importance'] = feature_importance
result_dict['prediction'] = prediction
if make_oof:
result_dict['oof'] = oof
return result_dict
submission.to_csv('submission.csv', index=False )<import_modules>
params = {'num_leaves': 512,
'objective': 'multiclass',
'max_depth': -1,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 3,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"random_state": 42,
"verbosity": -1,
"num_class": 5}
print(os.listdir(".. /input")) 
<load_from_csv>
result_dict_lgb = train_model(X=train, X_test=test, y=y, params=params, model_type='lgb', plot_feature_importance=True, make_oof=True )
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
train= pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv' )<prepare_x_and_y>
xgb_params = {'eta': 0.01, 'max_depth': 9, 'subsample': 0.9, 'colsample_bytree': 0.9,
'objective': 'multi:softprob', 'eval_metric': 'merror', 'silent': True, 'nthread': 4, 'num_class': 5}
result_dict_xgb = train_model(params=xgb_params, model_type='xgb', make_oof=True )
train["comment_text"].fillna("fillna")
 test["comment_text"].fillna("fillna")
 X_train = train["comment_text"].str.lower()
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test["comment_text"].str.lower()<define_variables>
prediction =(result_dict_lgb['prediction'] + result_dict_xgb['prediction'] ).argmax(1)
 submission = pd.DataFrame({'PetID': sub.PetID, 'AdoptionSpeed': [int(i)for i in prediction]})
 submission.head()
max_features=100000
maxlen=200
embed_size=300<compute_train_metric>
submission.to_csv('submission.csv', index=False )
<categorify><EOS> 
submission.to_csv('submission.csv', index=False )