Kaggle competition: Airbnb New User Bookings
def remove_char(text):
c = [''.join(i)for i in text if i not in punctuation]
return "".join(c)
 remove_char(s )<feature_engineering>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import accuracy_score
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
train.text = train.text.apply(remove_char )<feature_engineering>
train = pd.read_csv('/kaggle/input/airbnb-recruiting-new-user-bookings/train_users_2.csv.zip')
 test = pd.read_csv('/kaggle/input/airbnb-recruiting-new-user-bookings/test_users.csv.zip')
 age_gender = pd.read_csv('/kaggle/input/airbnb-recruiting-new-user-bookings/age_gender_bkts.csv.zip')
 sessions = pd.read_csv('/kaggle/input/airbnb-recruiting-new-user-bookings/sessions.csv.zip' )
test.text = test.text.apply(clean)
 test.text = test.text.apply(remove_char )<feature_engineering>
X_train = train.drop(['date_first_booking', 'country_destination'], axis=1)
 X_test = test.drop(['date_first_booking'], axis=1 )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
y_des = train['country_destination'].values
X=pd.concat(( X_train, X_test), axis=0, ignore_index=True)
 X.shape
train_input = bert_encode(train.text.values, tokenizer, max_len=150)
 test_input = bert_encode(test.text.values, tokenizer, max_len=150)
 train_labels = train.target.values<train_model>
X.loc[X.age.isnull() , 'age']=X.age.mean()
call = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=2, verbose=0, mode='auto', baseline=None, restore_best_weights=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=10,
batch_size=16,
callbacks=[call] 
)
 model.save('model.h5' )<predict_on_test>
dac = np.vstack(
 X.date_account_created.astype(str ).apply(
 lambda x: list(map(int, x.split('-')))
 ).values 
)
 X['dac_year'] = dac[:, 0]
X['dac_month'] = dac[:, 1]
X['dac_day'] = dac[:, 2]
X = X.drop(['date_account_created'], axis=1)
 X.head()
test_pred = model.predict(test_input )<save_to_csv>
df = sessions.user_id.value_counts()
print(df.shape)
 print(df.head() )
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )<load_from_csv>
df = df.to_frame()
df = df.rename(columns = {'user_id' : 'session_count'})
 df['id'] = df.index
df.head()
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 samp_sub = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv' )<import_modules>
X = pd.merge(X, df, how = 'left', on = ['id'])
 X.session_count.fillna(-1, inplace = True)
 X.session_count = X.session_count.astype(int )
!pip install pytorch-pretrained-bert pytorch-nlp 
<set_options>
tfa = np.vstack(
 X.timestamp_first_active.astype(str ).apply(
 lambda x: list(map(int, [x[:4], x[4:6], x[6:8],
x[8:10], x[10:12],
x[12:14]]))
).values 
)
 X['tfa_year'] = tfa[:, 0]
X['tfa_month'] = tfa[:, 1]
X['tfa_day'] = tfa[:, 2]
X = X.drop(['timestamp_first_active'], axis=1 )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 n_gpu = torch.cuda.device_count()
torch.cuda.get_device_name(0 )<define_variables>
oh_features = ['gender', 'signup_method', 'signup_flow', 'language',
'affiliate_channel', 'affiliate_provider',
'first_affiliate_tracked', 'signup_app',
'first_device_type', 'first_browser']
tweet_lens = [len(i)for i in train['text']]<define_variables>
for feature in oh_features:
X_dummy = pd.get_dummies(X[feature], prefix=feature)
 X = X.drop([feature], axis=1)
 X = pd.concat(( X, X_dummy), axis=1)
 X.head()
MAX_LEN = max(tweet_lens )<define_variables>
X_train = X.iloc[:len(train), :]
X_test = X.iloc[len(train):, :]
X_train = X_train.drop(['id'], axis=1)
 X_train.shape
X_test = X_test.drop(['id'], axis=1 )
labels = train['target'].values<define_variables>
le = LabelEncoder()
y_trans = le.fit_transform(y_des)
 y_trans.shape
nb_labels = len(labels )<define_variables>
dtrain, dtest, train_label, test_label = train_test_split(X_train, y_trans, test_size = 0.3, random_state = 817 )
sentences = ["[CLS] " + query + " [SEP]" for query in train['text']]
sentences_test = ["[CLS] " + query + " [SEP]" for query in test['text']]
print(sentences[0] )<string_transform>
logreg = LogisticRegression()
logreg.fit(dtrain, train_label)
 pred_log=logreg.predict(dtest)
 print(accuracy_score(test_label, pred_log))
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
 tokenized_texts = [tokenizer.tokenize(sent)for sent in sentences]
tokenized_texts_test = [tokenizer.tokenize(sent)for sent in sentences_test]
print("Tokenize the first sentence:")
 print(tokenized_texts[0] )<categorify>
rfc = RandomForestClassifier(max_depth=20, n_estimators=100)
 rfc.fit(dtrain , train_label)
 pred = rfc.predict(dtest)
 print(accuracy_score(test_label, pred))
input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt)for txt in tokenized_texts],
maxlen=MAX_LEN, dtype="long", truncating="post", padding="post")
 input_ids_test = pad_sequences([tokenizer.convert_tokens_to_ids(txt)for txt in tokenized_texts_test],
maxlen=MAX_LEN, dtype="long", truncating="post", padding="post" )<string_transform>
dtc = DecisionTreeClassifier(max_depth=10)
 dtc.fit(dtrain , train_label)
 pred = dtc.predict(dtest)
 print(accuracy_score(test_label, pred))
input_ids = [tokenizer.convert_tokens_to_ids(x)for x in tokenized_texts]
input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype="long", truncating="post", padding="post")
 input_ids_test = [tokenizer.convert_tokens_to_ids(x)for x in tokenized_texts_test]
input_ids_test = pad_sequences(input_ids_test, maxlen=MAX_LEN, dtype="long", truncating="post", padding="post" )<define_variables>
xgb = XGBClassifier(max_depth=4, learning_rate=0.03, n_estimators=100,
objective='multi:softprob', subsample=0.6, colsample_bytree=0.6, seed=40)
 xgb.fit(dtrain , train_label)
 pred = xgb.predict(dtest)
 print(accuracy_score(test_label, pred))
attention_masks = []
attention_masks_test = []
for seq in input_ids:
seq_mask = [float(i>0)for i in seq]
attention_masks.append(seq_mask)
 for seq in input_ids_test:
seq_mask = [float(i>0)for i in seq]
attention_masks_test.append(seq_mask)
<split>
xgb = XGBClassifier(max_depth=4, learning_rate=0.03, n_estimators=100,
objective='multi:softprob', subsample=0.6, colsample_bytree=0.6, seed=40)
 xgb.fit(X_train, y_trans)
 XGBC_pred_test = xgb.predict(X_test)
 XGBC_pred_test_prob=xgb.predict_proba(X_test )
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=2018, test_size=0.1)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, input_ids,
random_state=2018, test_size=0.1 )<data_type_conversions>
ids_test = test['id']
ids = []
countries = []
for i in range(len(X_test)) :
idx = ids_test[i]
ids += [idx] * 5
countries += le.inverse_transform(np.argsort(XGBC_pred_test_prob[i])[::-1][:5] ).tolist()
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels)
 validation_labels = torch.tensor(validation_labels)
 train_masks = torch.tensor(train_masks)
 validation_masks = torch.tensor(validation_masks)
<data_type_conversions>
submission = pd.DataFrame({
"id" : ids,
"country" : countries
})
 submission.to_csv('submission_XGBC.csv', index = False )
test_masks_sub = torch.tensor(attention_masks_test)
 test_inputs_sub = torch.tensor(input_ids_test)
 train_masks_sub = torch.tensor(attention_masks)
 train_inputs_sub = torch.tensor(input_ids)
 train_labels_sub = torch.tensor(labels )<define_variables>
n_labels=len(set(y_des))
n_labels
batch_size = 32<create_dataframe>
params = {
'objective': 'multi:softprob',
'eval_metric': 'merror',
'num_class': n_labels,
'eta': 0.5,
'max_depth': 6,
'subsample': 0.5,
'colsample_bytree': 0.3,
'silent': 1,
'seed': 123
}
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size )<load_pretrained>
num_boost_round = 50
Dtrain = xgb.DMatrix(X_train, y_trans)
 res = xgb.cv(params, Dtrain, num_boost_round=num_boost_round, nfold=5,
callbacks=[xgb.callback.print_evaluation(show_stdv=True),
xgb.callback.early_stop(50)] )
model = BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=nb_labels)
 model.cuda() 
<init_hyperparams>
num_boost_round = res['test-merror-mean'].idxmin()
print(format(num_boost_round))
clf = xgb.train(params, Dtrain, num_boost_round=num_boost_round)
 clf
param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay_rate': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay_rate': 0.0}
]<choose_model_class>
importance = clf.get_fscore()
importance_df = pd.DataFrame(
 sorted(importance.items() , key=operator.itemgetter(1)) ,
columns=['feature', 'fscore']
)
optimizer = BertAdam(optimizer_grouped_parameters,
lr=2e-5,
warmup=.1 )<compute_test_metric>
importance_df = importance_df.iloc[-20:, :]