Kaggle competition: Natural Language Processing with Disaster Tweets
submission = prediction(test_df=test,
test_audio=TEST_AUDIO_DIR,
model_config = model_config,
mel_params=settings['dataset']['params']['melspectrogram_parameters'],
resnest_melparams = melspectrogram_parameters,
b1_melparams = settings_effnetb1['dataset']['params']['melspectrogram_parameters'],
thefirst_melparams = settings_thefirst['dataset']['params']['melspectrogram_parameters'],
thesecond_melparams = settings_thesecond['dataset']['params']['melspectrogram_parameters'],
inception_melparams = settings_inception['dataset']['params']['melspectrogram_parameters'],
se_melparams = settings_se['dataset']['params']['melspectrogram_parameters'],
mixup_melparams=settings_mixup['dataset']['params']['melspectrogram_parameters'],
ls_melparams=settings_ls['dataset']['params']['melspectrogram_parameters'],
target_sr=TARGET_SR,
threshold=0.33,
maxpreds=3, 
)
 submission.to_csv("submission.csv", index=False )<import_modules>
stop_words = stopwords.words('english')
import cv2
import audioread
import logging
import os
import random
import time
import warnings
import librosa
import numpy as np
import pandas as pd
import soundfile as sf
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data as data
from contextlib import contextmanager
from pathlib import Path
from typing import Optional
from fastprogress import progress_bar
from sklearn.metrics import f1_score
from torchvision import models<set_options>
test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv" )
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_logger(out_file=None):
logger = logging.getLogger()
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 logger.handlers = []
logger.setLevel(logging.INFO)
 handler = logging.StreamHandler()
handler.setFormatter(formatter)
 handler.setLevel(logging.INFO)
 logger.addHandler(handler)
 if out_file is not None:
fh = logging.FileHandler(out_file)
 fh.setFormatter(formatter)
 fh.setLevel(logging.INFO)
 logger.addHandler(fh)
 logger.info("logger set up")
 return logger
@contextmanager
def timer(name: str, logger: Optional[logging.Logger] = None):
t0 = time.time()
msg = f"[{name}] start"
if logger is None:
print(msg)
 else:
logger.info(msg)
 yield
msg = f"[{name}] done in {time.time() - t0:.2f} s"
if logger is None:
print(msg)
 else:
logger.info(msg )<set_options>
print("Unique Keywords : {}, Unique locations : {}".format(len(train.keyword.unique()), len(train.location.unique())))
 train.keyword.unique()
logger = get_logger("main.log")
 set_seed(1213 )<define_variables>
print("Train :
")
 print(train.isnull().sum())
 print("
Test :
")
 print(test.isnull().sum() )
TARGET_SR = 32000
TEST = Path(".. /input/birdsong-recognition/test_audio" ).exists()<load_from_csv>
ydata = train["target"]
train = train.drop(["target","location", "id"], axis=1)
 test = test.drop(["id","location"], axis=1 )
if TEST:
DATA_DIR = Path(".. /input/birdsong-recognition/")
 else:
DATA_DIR = Path(".. /input/birdcall-check/")
 test = pd.read_csv(DATA_DIR / "test.csv")
 test_audio = DATA_DIR / "test_audio"
test.head()<save_to_csv>
keyword_mapper = {}
temp = train.keyword.unique()
for i in range(len(temp)) :
keyword_mapper[temp[i]] = i
train["keyword"] = train["keyword"].map(keyword_mapper)
 test["keyword"] = test["keyword"].map(keyword_mapper )
sub = pd.read_csv(".. /input/birdsong-recognition/sample_submission.csv")
 sub.to_csv("submission.csv", index=False )<choose_model_class>
def text_preprocessing(data):
data = data.apply(lambda x: x.strip().lower())
 data = data.apply(lambda x: re.sub(r'\d+', '', x))
data = data.apply(lambda x: x.translate(str.maketrans('', '', string.punctuation)))
 data = data.apply(lambda x : word_tokenize(x))
data = data.apply(lambda x: [word for word in x if word not in stop_words])
 lemmatizer = WordNetLemmatizer()
data = data.apply(lambda x: [lemmatizer.lemmatize(word, pos ='v')for word in x])
 return data
class ResNet(nn.Module):
def __init__(self, base_model_name: str, pretrained=False,
num_classes=264):
super().__init__()
base_model = models.__getattribute__(base_model_name )(
 pretrained=pretrained)
 layers = list(base_model.children())[:-2]
layers.append(nn.AdaptiveMaxPool2d(1))
self.encoder = nn.Sequential(*layers)
 in_features = base_model.fc.in_features
self.classifier = nn.Sequential(
 nn.Linear(in_features, 1024), nn.ReLU() , nn.Dropout(p=0.2),
nn.Linear(1024, 1024), nn.ReLU() , nn.Dropout(p=0.2),
nn.Linear(1024, num_classes))
def forward(self, x):
batch_size = x.size(0)
 x = self.encoder(x ).view(batch_size, -1)
 x = self.classifier(x)
 multiclass_proba = F.softmax(x, dim=1)
 multilabel_proba = F.sigmoid(x)
 return {
"logits": x,
"multiclass_proba": multiclass_proba,
"multilabel_proba": multilabel_proba
}<define_variables>
test["text"] = text_preprocessing(test["text"])
 train["text"] = text_preprocessing(train["text"] )
model_config = {
"base_model_name": "resnet50",
"pretrained": False,
"num_classes": 264
}
melspectrogram_parameters = {
"n_mels": 128,
"fmin": 20,
"fmax": 16000
}
weights_path = ".. /input/birdcall-resnet50-init-weights/best.pth"<define_variables>
vectorizer = TfidfVectorizer()
vector = vectorizer.fit_transform(["".join(i)for i in train["text"]])
 vector = vector.todense()
vector = np.concatenate(( vector, np.reshape(np.array(train["keyword"]),(train.keyword.shape[0],-1))), axis=1)
 print(vector.shape)
 vector_test = vectorizer.transform(["".join(i)for i in test["text"]])
 vector_test = vector_test.todense()
vector_test = np.concatenate(( vector_test, np.reshape(np.array(test["keyword"]),(test.keyword.shape[0],-1))), axis=1)
 print(vector_test.shape )
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }<choose_model_class>
xtrain, xtest, ytrain, ytest = train_test_split(vector, ydata, train_size = 0.75 )
def get_model(config: dict, weights_path: str):
model = ResNet(**config)
 checkpoint = torch.load(weights_path)
 model.load_state_dict(checkpoint["model_state_dict"])
 device = torch.device("cuda")
 model.to(device)
 model.eval()
return model<create_dataframe>
model = LinearSVC(loss="hinge",fit_intercept=False, max_iter=1500)
 model = model.fit(xtrain, ytrain)
 predictions = model.predict(xtest)
 print("Accuracy score: ", accuracy_score(ytest, predictions))
print("Precision score: ", precision_score(ytest, predictions))
print("Recall score: ", recall_score(ytest, predictions))
print("F1 score : ", f1_score(predictions, ytest))
confusion_matrix(ytest, predictions )
def prediction_for_clip(test_df: pd.DataFrame,
clip: np.ndarray,
model: ResNet,
mel_params: dict,
threshold=0.5):
dataset = TestDataset(df=test_df,
clip=clip,
img_size=224,
melspectrogram_parameters=mel_params)
 loader = data.DataLoader(dataset, batch_size=1, shuffle=False)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model.eval()
prediction_dict = {}
for image, row_id, site in progress_bar(loader):
site = site[0]
row_id = row_id[0]
if site in {"site_1", "site_2"}:
image = image.to(device)
 with torch.no_grad() :
prediction = model(image)
 proba = prediction["multilabel_proba"].detach().cpu().numpy().reshape(-1)
 events = proba >= threshold
labels = np.argwhere(events ).reshape(-1 ).tolist()
else:
image = image.squeeze(0)
 batch_size = 16
whole_size = image.size(0)
 if whole_size % batch_size == 0:
n_iter = whole_size // batch_size
else:
n_iter = whole_size // batch_size + 1
all_events = set()
for batch_i in range(n_iter):
batch = image[batch_i * batch_size:(batch_i + 1)* batch_size]
if batch.ndim == 3:
batch = batch.unsqueeze(0)
 batch = batch.to(device)
 with torch.no_grad() :
prediction = model(batch)
 proba = prediction["multilabel_proba"].detach().cpu().numpy()
events = proba >= threshold
for i in range(len(events)) :
event = events[i, :]
labels = np.argwhere(event ).reshape(-1 ).tolist()
for label in labels:
all_events.add(label)
 labels = list(all_events)
 if len(labels)== 0:
prediction_dict[row_id] = "nocall"
else:
labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))
label_string = " ".join(labels_str_list)
 prediction_dict[row_id] = label_string
return prediction_dict<load_pretrained>
model_lr = LogisticRegression(penalty='l2')
 model_lr = model_lr.fit(xtrain, ytrain)
 predictions = model_lr.predict(xtest)
 print("Accuracy score: ", accuracy_score(ytest, predictions))
print("Precision score: ", precision_score(ytest, predictions))
print("Recall score: ", recall_score(ytest, predictions))
print("F1 score : ", f1_score(predictions, ytest))
confusion_matrix(ytest, predictions )
def prediction(test_df: pd.DataFrame,
test_audio: Path,
model_config: dict,
mel_params: dict,
weights_path: str,
threshold=0.5):
model = get_model(model_config, weights_path)
 unique_audio_id = test_df.audio_id.unique()
warnings.filterwarnings("ignore")
 prediction_dfs = []
for audio_id in unique_audio_id:
with timer(f"Loading {audio_id}", logger):
clip, _ = librosa.load(test_audio /(audio_id + ".mp3"),
sr=TARGET_SR,
mono=True,
res_type="kaiser_fast")
 test_df_for_audio_id = test_df.query(
 f"audio_id == '{audio_id}'" ).reset_index(drop=True)
 with timer(f"Prediction on {audio_id}", logger):
prediction_dict = prediction_for_clip(test_df_for_audio_id,
clip=clip,
model=model,
mel_params=mel_params,
threshold=threshold)
 row_id = list(prediction_dict.keys())
 birds = list(prediction_dict.values())
 prediction_df = pd.DataFrame({
"row_id": row_id,
"birds": birds
})
 prediction_dfs.append(prediction_df)
 prediction_df = pd.concat(prediction_dfs, axis=0, sort=False ).reset_index(drop=True)
 return prediction_df<save_to_csv>
model_xgb = XGBClassifier(metric= 'euclidean', n_neighbors= 3, weights= 'uniform')
 model_xgb = model_xgb.fit(xtrain, ytrain)
 predictions = model_xgb.predict(xtest)
 print("Accuracy score: ", accuracy_score(ytest, predictions))
print("Precision score: ", precision_score(ytest, predictions))
print("Recall score: ", recall_score(ytest, predictions))
print("F1 score : ", f1_score(predictions, ytest))
confusion_matrix(ytest, predictions )
submission = prediction(test_df=test,
test_audio=test_audio,
model_config=model_config,
mel_params=melspectrogram_parameters,
weights_path=weights_path,
threshold=0.8)
 submission.to_csv("submission.csv", index=False )<load_from_csv>
tree = DecisionTreeClassifier(random_state = 11, max_features = "auto", class_weight = "balanced",max_depth = None)
 model_ada = AdaBoostClassifier(base_estimator=tree)
 model_ada = model_ada.fit(xtrain, ytrain)
 predictions = model_ada.predict(xtest)
 print("Accuracy score: ", accuracy_score(ytest, predictions))
print("Precision score: ", precision_score(ytest, predictions))
print("Recall score: ", recall_score(ytest, predictions))
print("F1 score : ", f1_score(predictions, ytest))
confusion_matrix(ytest, predictions )
train = pd.read_csv(".. /input/automation-of-feature-creation/train.csv")
 test = pd.read_csv(".. /input/automation-of-feature-creation/test.csv")
 tmp = pd.read_csv(".. /input/home-credit-default-risk/application_test.csv")
 tmp_train = pd.read_csv(".. /input/home-credit-default-risk/application_train.csv" )<feature_engineering>
model_gb = GradientBoostingClassifier(criterion='friedman_mse', learning_rate= 0.15,
loss= 'deviance', max_depth= 8, max_features='sqrt',
min_samples_leaf= 0.15714285714285714, min_samples_split= 0.5,
n_estimators= 10, subsample=1.0)
 model_gb = model_gb.fit(xtrain, ytrain)
 predictions = model_gb.predict(xtest)
 print("Accuracy score: ", accuracy_score(ytest, predictions))
print("Precision score: ", precision_score(ytest, predictions))
print("Recall score: ", recall_score(ytest, predictions))
print("F1 score : ", f1_score(predictions, ytest))
confusion_matrix(ytest, predictions )
train['SK_ID_CURR'] = tmp_train['SK_ID_CURR']
test['SK_ID_CURR'] = tmp['SK_ID_CURR']<prepare_x_and_y>
model = ExtraTreesClassifier(bootstrap=False, criterion='gini', max_depth= None,
max_features= 3, min_samples_leaf= 1, min_samples_split= 10,
n_estimators= 300)
 model.fit(xtrain, ytrain)
 predictions = model.predict(xtest)
 print("Accuracy score: ", accuracy_score(ytest, predictions))
print("Precision score: ", precision_score(ytest, predictions))
print("Recall score: ", recall_score(ytest, predictions))
print("F1 score : ", f1_score(predictions, ytest))
confusion_matrix(ytest, predictions )
probss = test['ProbTARGET1']
del test['ProbTARGET1']
del train['ProbTARGET1']
y = train['TARGET']
del train['TARGET']<feature_engineering>
y_pred = model_gb.predict(vector_test)
 sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 y_pred = np.round(y_pred ).astype(int ).reshape(3263)
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':y_pred})
 sub.to_csv('submission_gb.csv',index=False)
 y_pred = model_ada.predict(vector_test)
 sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 y_pred = np.round(y_pred ).astype(int ).reshape(3263)
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':y_pred})
 sub.to_csv('submission_ada.csv',index=False)
 y_pred = model_xgb.predict(vector_test)
 sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 y_pred = np.round(y_pred ).astype(int ).reshape(3263)
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':y_pred})
 sub.to_csv('submission_xgb.csv',index=False)
 y_pred = model_lr.predict(vector_test)
 sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 y_pred = np.round(y_pred ).astype(int ).reshape(3263)
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':y_pred})
 sub.to_csv('submission_lr.csv',index=False)
 y_pred = model.predict(vector_test)
 sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 y_pred = np.round(y_pred ).astype(int ).reshape(3263)
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':y_pred})
 sub.to_csv('submission_ebc.csv',index=False )
def mean_(x):
if '{' in x:
return x
if x=='Missing':
return -1
if '+inf' in x:
return float(x.replace(']','' ).replace('[','' ).split(';')[0])
 if '-inf' in x:
return float(x.replace(']','' ).replace('[','' ).split(';')[1])
 l = x.replace(']','' ).replace('[','' ).split(';')
 return(float(l[0])+float(l[1])) /2
dictionnary = {}
for i in train.columns:
if train[i].dtype!='object':
continue
try :
for j in train[i].unique() :
dictionnary[j] = mean_(j)
 except :
continue
for i in train.columns:
if train[i].dtype!='object':
continue
train[i] = train[i].map(dictionnary )<categorify>
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
for i in test.columns:
if test[i].dtype!='object':
continue
try :
for j in test[i].unique() :
dictionnary[j] = mean(j)
 except :
continue
for i in test.columns:
if test[i].dtype!='object':
continue
test[i] = test[i].map(dictionnary )<categorify>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
to_dummy =[]
for i in test.columns:
if test[i].dtype==object:
try :
test[i]=test[i].astype(float)
 except :
continue
if len(test[i].unique())<5:
to_dummy.append(i)
 else :
del test[i]
test = pd.get_dummies(test,columns=to_dummy)
 to_dummy =[]
for i in train.columns:
if train[i].dtype==object:
try :
train[i]=train[i].astype(float)
 except :
continue
if len(train[i].unique())<5:
to_dummy.append(i)
 else :
del train[i]
train = pd.get_dummies(train,columns=to_dummy)
<train_model>
import tokenization
train.fillna(0,inplace=True)
 test.fillna(0,inplace=True )<rename_columns>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
col_dict = {}
for i in train.columns:
if '>' in i:
col_dict[i] = i.replace('>','' ).replace('<','')
 train.rename(columns=col_dict,inplace = True)
 test.rename(columns=col_dict, inplace =True )<drop_column>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
for i in train.columns:
if train[i].dtype==object:
del train[i]
for i in test.columns:
if test[i].dtype==object:
del test[i]
for i in train.columns:
if i not in test.columns:
del train[i]
for j in test.columns:
if j not in train.columns:
del test[j]<train_model>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
def train_model(data_, test_, y_, folds_):
oof_preds = np.zeros(data_.shape[0])
 sub_preds = np.zeros(test_.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in data_.columns if f not in ['SK_ID_CURR']]
for n_fold,(trn_idx, val_idx)in enumerate(folds_.split(data_)) :
trn_x, trn_y = data_[feats].iloc[trn_idx], y_.iloc[trn_idx]
val_x, val_y = data_[feats].iloc[val_idx], y_.iloc[val_idx]
clf = LGBMClassifier(
 n_estimators=6000,
learning_rate=0.02,
num_leaves=40,
colsample_bytree=.8,
subsample=.8,
max_depth=12,
reg_alpha=.1,
reg_lambda=.1,
min_split_gain=.008,
min_child_weight=2,
min_child_samples=35,
silent=-1,
verbose=-1, 
)
 clf.fit(trn_x, trn_y,
eval_set= [(trn_x, trn_y),(val_x, val_y)],
eval_metric='auc', verbose=500, early_stopping_rounds=300 
)
 oof_preds[val_idx] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_[feats], num_iteration=clf.best_iteration_)[:, 1] / folds_.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))
 del clf, trn_x, trn_y, val_x, val_y
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(y, oof_preds))
test_['TARGET'] = sub_preds
return oof_preds, test_[['SK_ID_CURR', 'TARGET']], feature_importance_df
def display_importances(feature_importance_df_):
cols = feature_importance_df_[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
plt.figure(figsize=(8,10))
sns.barplot(x="importance", y="feature",
data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features(avg over folds)')
 plt.savefig('lgbm_importances.png')
 def display_roc_curve(y_, oof_preds_, folds_idx_):
plt.figure(figsize=(6,6))
scores = []
for n_fold,(_, val_idx)in enumerate(folds_idx_):
fpr, tpr, thresholds = roc_curve(y_.iloc[val_idx], oof_preds_[val_idx])
 score = roc_auc_score(y_.iloc[val_idx], oof_preds_[val_idx])
 scores.append(score)
 plt.plot(fpr, tpr, lw=1, alpha=0.3, label='ROC fold %d(AUC = %0.4f)' %(n_fold + 1, score))
plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r', label='Luck', alpha=.8)
 fpr, tpr, thresholds = roc_curve(y_, oof_preds_)
 score = roc_auc_score(y_, oof_preds_)
 plt.plot(fpr, tpr, color='b',
label='Avg ROC(AUC = %0.4f $\pm$ %0.4f)' %(score, np.std(scores)) ,
lw=2, alpha=.8)
 plt.xlim([-0.05, 1.05])
 plt.ylim([-0.05, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title('LightGBM ROC Curve')
 plt.legend(loc="lower right")
 plt.savefig('roc_curve.png')
 def display_precision_recall(y_, oof_preds_, folds_idx_):
plt.figure(figsize=(6,6))
scores = []
for n_fold,(_, val_idx)in enumerate(folds_idx_):
fpr, tpr, thresholds = roc_curve(y_.iloc[val_idx], oof_preds_[val_idx])
 score = average_precision_score(y_.iloc[val_idx], oof_preds_[val_idx])
 scores.append(score)
 plt.plot(fpr, tpr, lw=1, alpha=0.3, label='AP fold %d(AUC = %0.4f)' %(n_fold + 1, score))
precision, recall, thresholds = precision_recall_curve(y_, oof_preds_)
 score = average_precision_score(y_, oof_preds_)
 plt.plot(precision, recall, color='b',
label='Avg ROC(AUC = %0.4f $\pm$ %0.4f)' %(score, np.std(scores)) ,
lw=2, alpha=.8)
 plt.xlim([-0.05, 1.05])
 plt.ylim([-0.05, 1.05])
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.title('LightGBM Recall / Precision')
 plt.legend(loc="best")
 plt.savefig('recall_precision_curve.png' )<save_to_csv>
train_input = bert_encode([" ".join(i)for i in train.text], tokenizer, max_len=160)
 test_input = bert_encode([" ".join(i)for i in test.text], tokenizer, max_len=160 )
if __name__ == '__main__':
gc.enable()
folds = KFold(n_splits=5, shuffle=True, random_state=123)
 oof_preds, test_preds, importances = train_model(train, test, y, folds)
 test_preds.to_csv('first_automated_submission.csv', index=False)
 folds_idx = [(trn_idx, val_idx)for trn_idx, val_idx in folds.split(train)]
display_importances(feature_importance_df_=importances)
 display_roc_curve(y_=y, oof_preds_=oof_preds, folds_idx_=folds_idx)
 display_precision_recall(y_=y, oof_preds_=oof_preds, folds_idx_=folds_idx )<import_modules>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model
import tensorflow as tf<load_from_csv>
early = EarlyStopping(monitor='val_loss',mode='auto', baseline=None, restore_best_weights=False)
 train_history = model.fit(
 train_input, ydata,
validation_split=0.2,
epochs=100,
batch_size=16,
callbacks=[early], verbose=1 
)
 model.save('model.h5' )
<load_from_csv><EOS> 
predictions = model.predict(test_input)
 for i in [0.5, 0.6, 0.4]:
arr = [1 if j>i else 0 for j in predictions]
sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':arr})
 sub.to_csv('submission_bert'+str(i)+'.csv',index=False )