Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<import_modules>
%matplotlib inline
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore' )<load_from_csv>
file = '.. /input/train.csv'
df = pd.read_csv(file )
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 print('Training data shape: ', train.shape)
 train.head()<load_from_csv>
df = df.drop(['PassengerId', 'Name', 'Ticket'], axis=1 )
test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 print('Testing data shape: ', test.shape)
 test.head()<count_missing_values>
df['Survived'].value_counts()
train.isnull().sum()<count_missing_values>
df['SibSp'].value_counts()
test.isnull().sum()<count_values>
df.isnull().sum(axis = 0 )
train['target'].value_counts()<create_dataframe>
df_hot = pd.concat([df, pd.get_dummies(df['Sex'])], axis=1)
 df_hot = df_hot.drop(['Sex'], axis=1 )
train1 = train.copy()
test1 = test.copy()<categorify>
df_hot = pd.concat([df_hot, pd.get_dummies(df['Pclass'])], axis=1)
 df_hot = df_hot.drop(['Pclass'], axis=1 )
def clean_text(text):

 text = text.lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 text = re.sub('[‘’“”…]', '', text)
 return text<drop_column>
df_hot = df_hot.rename(columns={df_hot.columns[9]:'Pclass1', df_hot.columns[10]:'Pclass2', df_hot.columns[11]:'Pclass3'} )
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train1['text'] = train1['text'].apply(lambda x: remove_emoji(x))
test1['text'] = test1['text'].apply(lambda x: remove_emoji(x))<feature_engineering>
df_hot['Embarked'].value_counts()
def text_preprocessing(text):

 tokenizer_reg = nltk.tokenize.RegexpTokenizer(r'\w+')
 nopunc = clean_text(text)
 tokenized_text = tokenizer_reg.tokenize(nopunc)
 remove_stopwords = [w for w in tokenized_text if w not in stopwords.words('english')]
combined_text = ' '.join(remove_stopwords)
 return combined_text
train1['text'] = train1['text'].apply(lambda x: text_preprocessing(x))
test1['text'] = test1['text'].apply(lambda x: text_preprocessing(x))
train1['text'].head()<feature_engineering>
df_hot['Embarked'].fillna("S", inplace = True )
count_vectorizer = CountVectorizer(ngram_range =(1,1), min_df = 1)
 train_vectors = count_vectorizer.fit_transform(train1['text'])
 test_vectors = count_vectorizer.transform(test1["text"])
 train_vectors.shape<categorify>
pd.options.display.float_format = '{:.2f}'.format
le = LabelEncoder()
T = df_hot['Embarked']
encoded = le.fit_transform(np.ravel(T))
tfidf = TfidfVectorizer(ngram_range=(1, 2), min_df = 2, max_df = 0.5)
 train_tfidf = tfidf.fit_transform(train1['text'])
 test_tfidf = tfidf.transform(test1["text"])
 train_tfidf.shape<compute_train_metric>
df_hot = pd.concat([df_hot, pd.DataFrame(encoded, columns = ['Emb'])], axis=1)
 df_hot = df_hot.drop(['Embarked'], axis=1 )
logreg_bow = LogisticRegression(C=1.0)
 scores = model_selection.cross_val_score(logreg_bow, train_vectors, train["target"], cv=5, scoring="f1")
 scores.mean()<compute_train_metric>
df_hot = df_hot[['Survived', 'SibSp', 'Parch', 'Fare', 'Emb', 'Cabin', 'Age',
'female', 'male', 'Pclass1', 'Pclass2', 'Pclass3']]
logreg_tfidf = LogisticRegression(C=1.0)
 scores = model_selection.cross_val_score(logreg_tfidf, train_tfidf, train["target"], cv=5, scoring="f1")
 scores.mean()<compute_train_metric>
df_hot = df_hot.drop(['Cabin'], axis=1 )
NB_bow = MultinomialNB()
scores = model_selection.cross_val_score(NB_bow, train_vectors, train["target"], cv=5, scoring="f1")
 scores.mean()<compute_train_metric>
train_cols = list(df_hot )
NB_tfidf = MultinomialNB()
scores = model_selection.cross_val_score(NB_tfidf, train_tfidf, train["target"], cv=5, scoring="f1")
 scores.mean()<train_model>
dfi = pd.DataFrame(KNN(k=5 ).fit_transform(df_hot))
dfi.columns = train_cols
NB_bow.fit(train_vectors, train["target"] )<save_to_csv>
dfi.isnull().sum()
sample_submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 sample_submission["target"] = NB_bow.predict(test_vectors)
 os.chdir('/kaggle/working')
 sample_submission.to_csv("submission1.csv", index=False )<categorify>
df1 = df.drop(['Age'], axis=1)
 df1 = pd.concat([df1, dfi['Age']], axis=1 )
def bert_encode(texts, tokenizer, max_len = 512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
df1['Embarked'].fillna("S", inplace = True )
def build_model(bert_layer, max_len = 512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_from_csv>
df1 = df1.drop(['Cabin'], axis=1 )
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )<categorify>
le = LabelEncoder()
T = df1['Sex']
encoded = le.fit_transform(np.ravel(T))
<choose_model_class>
df1 = pd.concat([df1, pd.DataFrame(encoded, columns = ['Gender'])], axis=1)
 df1 = df1.drop(['Sex'], axis=1 )
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<feature_engineering>
le = LabelEncoder()
T = df1['Embarked']
encoded = le.fit_transform(np.ravel(T))
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
df1 = pd.concat([df1, pd.DataFrame(encoded, columns = ['Emb'])], axis=1)
 df1 = df1.drop(['Embarked'], axis=1 )
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values<train_model>
df1.isnull().sum()
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
callbacks=[checkpoint],
batch_size=16
)<predict_on_test>
model.load_weights('model.h5')
 test_pred = model.predict(test_input )<save_to_csv>
X = np.asarray(dfi.drop(['Survived'], axis = 1))
y = np.asarray(dfi[['Survived']] )
submission['target'] = test_pred.round().astype(int)
 os.chdir('/kaggle/working')
 submission.to_csv("submission2.csv", index=False )<define_variables>
X1 = np.asarray(df1.drop(['Survived'], axis = 1))
y1 = np.asarray(df1[['Survived']] )
%ls /kaggle/input/nlp-getting-started/<define_variables>
X = preprocessing.StandardScaler().fit(X ).transform(X )
prefix = '/kaggle/input/nlp-getting-started/'<load_from_csv>
X1 = preprocessing.StandardScaler().fit(X1 ).transform(X1 )
train_df = pd.read_csv(prefix + 'train.csv')
<prepare_output>
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn import metrics
train_df = pd.DataFrame({
'id':range(len(train_df)) ,
'label':train_df["target"],
'alpha':['a']*train_df.shape[0],
'text': train_df["text"].replace(r'
', ' ', regex=True)
 })
 train_df.head()
len(train_df )<load_from_csv>
GNB = GaussianNB()
scores_GNB = cross_val_score(GNB, X, y.ravel() , cv=100, scoring = "accuracy" )
test_df = pd.read_csv(prefix + 'test.csv')
<count_missing_values>
LR = LogisticRegression(solver = 'lbfgs', max_iter=1000)
 scores_LR = cross_val_score(LR, X, y.ravel() , cv=100, scoring = "accuracy" )
test_df.isnull().sum()<define_variables>
RF = RandomForestClassifier(n_estimators=100, random_state=0)
 scores_RF = cross_val_score(RF, X1, y1.ravel() , cv=100, scoring = "accuracy" )
test_df = pd.DataFrame({
'id':range(len(test_df)) ,
'label':[0]*test_df.shape[0],
'alpha':['a']*test_df.shape[0],
'text': test_df["text"].replace(r'
', ' ', regex=True)
 })
 test_df.head()
len(test_df )<install_modules>
VM = SVC(C = 0.01, kernel = 'rbf', gamma = 'scale', class_weight = 'balanced')
 scores_SVC = cross_val_score(VM, X1, y1.ravel() , cv=100, scoring = "accuracy" )
!mkdir data
!pip install contractions<feature_engineering>
NGH = KNeighborsClassifier(n_neighbors = 2)
 scores_KNN = cross_val_score(NGH, X, y.ravel() , cv=100, scoring = "accuracy" )
def fix_contractions(text):
return contractions.fix(text)
 def remove_url(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_mark(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 print("tweet before contractions fix : ", train_df.iloc[1055]["text"])
 print("-"*20)
 train_df['text']=train_df['text'].apply(lambda x : fix_contractions(x))
test_df['text']=test_df['text'].apply(lambda x : fix_contractions(x))
train_df['text']=train_df['text'].apply(lambda x : remove_url(x))
test_df['text']=test_df['text'].apply(lambda x : remove_url(x))
train_df['text']=train_df['text'].apply(lambda x : remove_mark(x))
test_df['text']=test_df['text'].apply(lambda x : remove_mark(x))
print("tweet after contractions fix : ", train_df.iloc[1055]["text"])
 train_df.to_csv('data/train.tsv', sep='\t', index=False, header=False)
 test_df.to_csv('data/dev.tsv', sep='\t', index=False, header=False )<import_modules>
BST = XGBClassifier()
scores_XGB = cross_val_score(BST, X, y.ravel() , cv=10, scoring = "accuracy" )
logger = logging.getLogger(__name__)
 csv.field_size_limit(2147483647)
 class InputExample(object):

 def __init__(self, guid, text_a, text_b=None, label=None):

 self.guid = guid
self.text_a = text_a
self.text_b = text_b
self.label = label
class InputFeatures(object):

 def __init__(self, input_ids, input_mask, segment_ids, label_id):
self.input_ids = input_ids
self.input_mask = input_mask
self.segment_ids = segment_ids
self.label_id = label_id
class DataProcessor(object):

 def get_train_examples(self, data_dir):

 raise NotImplementedError()
def get_dev_examples(self, data_dir):

 raise NotImplementedError()
def get_labels(self):

 raise NotImplementedError()
@classmethod
def _read_tsv(cls, input_file, quotechar=None):

 with open(input_file, "r", encoding="utf-8-sig")as f:
reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
 lines = []
for line in reader:
if sys.version_info[0] == 2:
line = list(unicode(cell, 'utf-8')for cell in line)
 lines.append(line)
 return lines
class BinaryProcessor(DataProcessor):

 def get_train_examples(self, data_dir):

 return self._create_examples(
 self._read_tsv(os.path.join(data_dir, "train.tsv")) , "train")
 def get_dev_examples(self, data_dir):

 return self._create_examples(
 self._read_tsv(os.path.join(data_dir, "dev.tsv")) , "dev")
 def get_labels(self):

 return ["0", "1"]
def _create_examples(self, lines, set_type):

 examples = []
for(i, line)in enumerate(lines):
guid = "%s-%s" %(set_type, i)
 text_a = line[3]
label = line[1]
examples.append(
 InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
return examples
def convert_example_to_feature(example_row, pad_token=0,
sequence_a_segment_id=0, sequence_b_segment_id=1,
cls_token_segment_id=1, pad_token_segment_id=0,
mask_padding_with_zero=True):
example, label_map, max_seq_length, tokenizer, output_mode, cls_token_at_end, cls_token, sep_token, cls_token_segment_id, pad_on_left, pad_token_segment_id = example_row
tokens_a = tokenizer.tokenize(example.text_a)
 tokens_b = None
if example.text_b:
tokens_b = tokenizer.tokenize(example.text_b)
 _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
 else:
if len(tokens_a)> max_seq_length - 2:
tokens_a = tokens_a[:(max_seq_length - 2)]
tokens = tokens_a + [sep_token]
segment_ids = [sequence_a_segment_id] * len(tokens)
 if tokens_b:
tokens += tokens_b + [sep_token]
segment_ids += [sequence_b_segment_id] *(len(tokens_b)+ 1)
 if cls_token_at_end:
tokens = tokens + [cls_token]
segment_ids = segment_ids + [cls_token_segment_id]
else:
tokens = [cls_token] + tokens
segment_ids = [cls_token_segment_id] + segment_ids
input_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
 padding_length = max_seq_length - len(input_ids)
 if pad_on_left:
input_ids =([pad_token] * padding_length)+ input_ids
input_mask =([0 if mask_padding_with_zero else 1] * padding_length)+ input_mask
segment_ids =([pad_token_segment_id] * padding_length)+ segment_ids
else:
input_ids = input_ids +([pad_token] * padding_length)
 input_mask = input_mask +([0 if mask_padding_with_zero else 1] * padding_length)
 segment_ids = segment_ids +([pad_token_segment_id] * padding_length)
 assert len(input_ids)== max_seq_length
assert len(input_mask)== max_seq_length
assert len(segment_ids)== max_seq_length
if output_mode == "classification":
label_id = label_map[example.label]
elif output_mode == "regression":
label_id = float(example.label)
 else:
raise KeyError(output_mode)
 return InputFeatures(input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
label_id=label_id)
 def convert_examples_to_features(examples, label_list, max_seq_length,
tokenizer, output_mode,
cls_token_at_end=False, pad_on_left=False,
cls_token='[CLS]', sep_token='[SEP]', pad_token=0,
sequence_a_segment_id=0, sequence_b_segment_id=1,
cls_token_segment_id=1, pad_token_segment_id=0,
mask_padding_with_zero=True):

 label_map = {label : i for i, label in enumerate(label_list)}
examples = [(example, label_map, max_seq_length, tokenizer, output_mode, cls_token_at_end, cls_token, sep_token, cls_token_segment_id, pad_on_left, pad_token_segment_id)for example in examples]
process_count = 1
with Pool(process_count)as p:
features = list(tqdm(p.imap(convert_example_to_feature, examples, chunksize=100), total=len(examples)))
 return features
def _truncate_seq_pair(tokens_a, tokens_b, max_length):

 while True:
total_length = len(tokens_a)+ len(tokens_b)
 if total_length <= max_length:
break
if len(tokens_a)> len(tokens_b):
tokens_a.pop()
else:
tokens_b.pop()
processors = {
"binary": BinaryProcessor
}
output_modes = {
"binary": "classification"
}
GLUE_TASKS_NUM_LABELS = {
"binary": 2
}<install_modules>
result_df = pd.DataFrame(columns=['Accuracy','Variance'], index=['Naive Bayes', 'Logistic Regression', 'Random Forest', 'SVC', 'KNN', 'XGBoost'] )
!pip install pytorch_transformers<import_modules>
result_df.iloc[0] = pd.Series({'Accuracy':scores_GNB.mean() ,
'Variance':np.std(scores_GNB)})
 result_df.iloc[1] = pd.Series({'Accuracy':scores_LR.mean() ,
'Variance':np.std(scores_LR)})
 result_df.iloc[2] = pd.Series({'Accuracy':scores_RF.mean() ,
'Variance':np.std(scores_RF)})
 result_df.iloc[3] = pd.Series({'Accuracy':scores_SVC.mean() ,
'Variance':np.std(scores_SVC)})
 result_df.iloc[4] = pd.Series({'Accuracy':scores_KNN.mean() ,
'Variance':np.std(scores_KNN)})
 result_df.iloc[5] = pd.Series({'Accuracy':scores_XGB.mean() ,
'Variance':np.std(scores_XGB)})
TensorDataset)
 XLMConfig, XLMForSequenceClassification, XLMTokenizer,
XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer,
RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer)
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__ )<define_variables>
result_df = result_df.sort_values('Accuracy', ascending = False, axis=0)
 result_df.head(6 )
args = {
'data_dir': './data/',
'model_type': 'bert',
'model_name': 'bert-base-cased',
'task_name': 'binary',
'output_dir': 'outputs/',
'cache_dir': 'cache/',
'do_train': True,
'do_eval': True,
'fp16': False,
'fp16_opt_level': 'O1',
'max_seq_length': 128,
'output_mode': 'classification',
'train_batch_size': 8,
'eval_batch_size': 8,
'gradient_accumulation_steps': 1,
'num_train_epochs': 3,
'weight_decay': 0,
'learning_rate': 4e-5,
'adam_epsilon': 1e-8,
'warmup_steps': 0,
'max_grad_norm': 1.0,
'logging_steps': 50,
'evaluate_during_training': False,
'save_steps': 2000,
'eval_all_checkpoints': True,
'overwrite_output_dir': False,
'reprocess_input_data': True,
'notes': 'Using Yelp Reviews dataset'
}
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<define_variables>
X_train, X_test, y_train, y_test = train_test_split(X, y.ravel() , test_size=0.2, random_state=4)
 print('Train set:', X_train.shape, y_train.shape)
 print('Test set:', X_test.shape, y_test.shape )
MODEL_CLASSES = {
'bert':(BertConfig, BertForSequenceClassification, BertTokenizer),
'xlnet':(XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),
'xlm':(XLMConfig, XLMForSequenceClassification, XLMTokenizer),
'roberta':(RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer)
 }
config_class, model_class, tokenizer_class = MODEL_CLASSES[args['model_type']]<load_pretrained>
cls = XGBClassifier()
cls.fit(X_train, y_train )
config = config_class.from_pretrained(args['model_name'], num_labels=2, finetuning_task=args['task_name'])
 tokenizer = tokenizer_class.from_pretrained(args['model_name'] )<load_pretrained>
file3 = '.. /input/test.csv'
dft = pd.read_csv(file3 )
model = model_class.from_pretrained(args['model_name'] )<train_model>
dft = dft.drop(['PassengerId', 'Name', 'Ticket'], axis=1 )
model.to(device )<categorify>
dft_hot = pd.concat([dft, pd.get_dummies(dft['Sex'])], axis=1)
 dft_hot = dft_hot.drop(['Sex'], axis=1 )
task = args['task_name']
processor = processors[task]()
label_list = processor.get_labels()
num_labels = len(label_list )<load_pretrained>
dft_hot = pd.concat([dft_hot, pd.get_dummies(dft['Pclass'])], axis=1)
 dft_hot = dft_hot.drop(['Pclass'], axis=1 )
def load_and_cache_examples(task, tokenizer, evaluate=False):
processor = processors[task]()
output_mode = args['output_mode']
mode = 'dev' if evaluate else 'train'
cached_features_file = os.path.join(args['data_dir'], f"cached_{mode}_{args['model_name']}_{args['max_seq_length']}_{task}")
 if os.path.exists(cached_features_file)and not args['reprocess_input_data']:
logger.info("Loading features from cached file %s", cached_features_file)
 features = torch.load(cached_features_file)
 else:
logger.info("Creating features from dataset file at %s", args['data_dir'])
 label_list = processor.get_labels()
examples = processor.get_dev_examples(args['data_dir'])if evaluate else processor.get_train_examples(args['data_dir'])
 features = convert_examples_to_features(examples, label_list, args['max_seq_length'], tokenizer, output_mode,
cls_token_at_end=bool(args['model_type'] in ['xlnet']),
cls_token=tokenizer.cls_token,
sep_token=tokenizer.sep_token,
cls_token_segment_id=2 if args['model_type'] in ['xlnet'] else 0,
pad_on_left=bool(args['model_type'] in ['xlnet']),
pad_token_segment_id=4 if args['model_type'] in ['xlnet'] else 0)
 logger.info("Saving features into cached file %s", cached_features_file)
 torch.save(features, cached_features_file)
 all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
 all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
 all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
 if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
 elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float)
 dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
 return dataset<train_model>
dft_hot = dft_hot.rename(columns={dft_hot.columns[8]:'Pclass1', dft_hot.columns[9]:'Pclass2', dft_hot.columns[10]:'Pclass3'} )
def train(train_dataset, model, tokenizer):
tb_writer = SummaryWriter()
train_sampler = RandomSampler(train_dataset)
 train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args['train_batch_size'])
 t_total = len(train_dataloader)// args['gradient_accumulation_steps'] * args['num_train_epochs']
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args['weight_decay']},
{'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
optimizer = AdamW(optimizer_grouped_parameters, lr=args['learning_rate'], eps=args['adam_epsilon'])
 scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args['warmup_steps'], t_total=t_total)
 if args['fp16']:
try:
except ImportError:
raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
 model, optimizer = amp.initialize(model, optimizer, opt_level=args['fp16_opt_level'])
 logger.info("***** Running training *****")
 logger.info(" Num examples = %d", len(train_dataset))
logger.info(" Num Epochs = %d", args['num_train_epochs'])
 logger.info(" Total train batch size = %d", args['train_batch_size'])
 logger.info(" Gradient Accumulation steps = %d", args['gradient_accumulation_steps'])
 logger.info(" Total optimization steps = %d", t_total)
 global_step = 0
tr_loss, logging_loss = 0.0, 0.0
model.zero_grad()
train_iterator = trange(int(args['num_train_epochs']), desc="Epoch")
 for _ in train_iterator:
epoch_iterator = tqdm_notebook(train_dataloader, desc="Iteration")
 for step, batch in enumerate(epoch_iterator):
model.train()
batch = tuple(t.to(device)for t in batch)
 inputs = {'input_ids': batch[0],
'attention_mask': batch[1],
'token_type_ids': batch[2] if args['model_type'] in ['bert', 'xlnet'] else None,
'labels': batch[3]}
outputs = model(input_ids=batch[0],attention_mask=batch[1],token_type_ids=batch[2],labels=batch[3])
 loss = outputs[0]
print("\r%f" % loss, end='')
 if args['gradient_accumulation_steps'] > 1:
loss = loss / args['gradient_accumulation_steps']
if args['fp16']:
with amp.scale_loss(loss, optimizer)as scaled_loss:
scaled_loss.backward()
torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args['max_grad_norm'])
 else:
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , args['max_grad_norm'])
 tr_loss += loss.item()
if(step + 1)% args['gradient_accumulation_steps'] == 0:
scheduler.step()
optimizer.step()
model.zero_grad()
global_step += 1
if args['logging_steps'] > 0 and global_step % args['logging_steps'] == 0:
if args['evaluate_during_training']:
results = evaluate(model, tokenizer)
 for key, value in results.items() :
tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
 tb_writer.add_scalar('lr', scheduler.get_lr() [0], global_step)
 tb_writer.add_scalar('loss',(tr_loss - logging_loss)/args['logging_steps'], global_step)
 logging_loss = tr_loss
if args['save_steps'] > 0 and global_step % args['save_steps'] == 0:
output_dir = os.path.join(args['output_dir'], 'checkpoint-{}'.format(global_step))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
 model_to_save = model.module if hasattr(model, 'module')else model
model_to_save.save_pretrained(output_dir)
 logger.info("Saving model checkpoint to %s", output_dir)
 return global_step, tr_loss / global_step<train_model>
le = LabelEncoder()
T = dft_hot['Embarked']
encoded = le.fit_transform(np.ravel(T))
if args['do_train']:
train_dataset = load_and_cache_examples(task, tokenizer)
 global_step, tr_loss = train(train_dataset, model, tokenizer)
 logger.info(" global_step = %s, average loss = %s", global_step, tr_loss )<save_to_csv>
dft_hot = pd.concat([dft_hot, pd.DataFrame(encoded, columns = ['Emb'])], axis=1)
 dft_hot = dft_hot.drop(['Embarked'], axis=1 )
def submit(pred_ids):
sub = pd.read_csv(prefix+'sample_submission.csv')
 sub['target'] = list(map(int,pred_ids))
sub.to_csv('submission.csv', index=False )<compute_train_metric>
dft_hot = dft_hot[['SibSp', 'Parch', 'Fare', 'Emb', 'Cabin', 'Age',
'female', 'male', 'Pclass1', 'Pclass2', 'Pclass3']]
def get_mismatched(labels, preds):
mismatched = labels != preds
examples = processor.get_dev_examples(args['data_dir'])
 wrong = [i for(i, v)in zip(examples, mismatched)if v]
return wrong
def get_eval_report(labels, preds):
mcc = matthews_corrcoef(labels, preds)
 tn, fp, fn, tp = confusion_matrix(labels, preds ).ravel()
return {
"mcc": mcc,
"tp": tp,
"tn": tn,
"fp": fp,
"fn": fn
}, get_mismatched(labels, preds)
 def compute_metrics(task_name, preds, labels):
assert len(preds)== len(labels)
 return get_eval_report(labels, preds)
 def evaluate(model, tokenizer, prefix=""):
eval_output_dir = args['output_dir']
results = {}
EVAL_TASK = args['task_name']
eval_dataset = load_and_cache_examples(EVAL_TASK, tokenizer, evaluate=True)
 if not os.path.exists(eval_output_dir):
os.makedirs(eval_output_dir)
 eval_sampler = SequentialSampler(eval_dataset)
 eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args['eval_batch_size'])
 preds_ = []
logger.info("***** Running evaluation {} *****".format(prefix))
logger.info(" Num examples = %d", len(eval_dataset))
logger.info(" Batch size = %d", args['eval_batch_size'])
 eval_loss = 0.0
nb_eval_steps = 0
preds = None
out_label_ids = None
preds_ = []
for batch in tqdm_notebook(eval_dataloader, desc="Evaluating"):
model.eval()
batch = tuple(t.to(device)for t in batch)
 with torch.no_grad() :
inputs = {'input_ids': batch[0],
'attention_mask': batch[1],
'token_type_ids': batch[2] if args['model_type'] in ['bert', 'xlnet'] else None,
'labels': batch[3]}
outputs = model(**inputs)
 tmp_eval_loss, logits = outputs[:2]
for i in range(logits.size(0)) :
preds_.append(logits[i,:].cpu().max(0)[1].item())
 eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if preds is None:
preds = logits.detach().cpu().numpy()
out_label_ids = inputs['labels'].detach().cpu().numpy()
else:
preds = np.append(preds, logits.detach().cpu().numpy() , axis=0)
 out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy() , axis=0)
 submit(preds_)
 eval_loss = eval_loss / nb_eval_steps
if args['output_mode'] == "classification":
preds = np.argmax(preds, axis=1)
 elif args['output_mode'] == "regression":
preds = np.squeeze(preds)
 result, wrong = compute_metrics(EVAL_TASK, preds, out_label_ids)
 results.update(result)
 output_eval_file = os.path.join(eval_output_dir, "eval_results.txt")
 with open(output_eval_file, "w")as writer:
logger.info("***** Eval results {} *****".format(prefix))
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s
" %(key, str(result[key])))
 return results, wrong<load_from_csv>
dft_hot = dft_hot.drop(['Cabin'], axis=1 )
<import_modules>
train_cols = list(dft_hot)
nltk.download('stopwords')
 stop_words = stopwords.words('english')
 nlp = spacy.load('en' )<install_modules>
dft_i = pd.DataFrame(KNN(k=5 ).fit_transform(dft_hot))
dft_i.columns = train_cols
!pip install transformers<import_modules>
X_test = np.asarray(dft_i )
from transformers import RobertaModel, RobertaTokenizer
from transformers import RobertaForSequenceClassification, RobertaConfig, AdamW, get_linear_schedule_with_warmup<set_options>
X_test = preprocessing.StandardScaler().fit(X_test ).transform(X_test )
print(torch.cuda.is_available() )<load_from_csv>
y_pred = cls.predict(X_test )
data = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")[["text", "target"]]
test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 data.head()<categorify>
dft2 = pd.read_csv(file3 )
def clean_text(text):
text = re.sub(r"\x89Û_", "", text)
 text = re.sub(r"\x89ÛÒ", "", text)
 text = re.sub(r"\x89ÛÓ", "", text)
 text = re.sub(r"\x89ÛÏWhen", "When", text)
 text = re.sub(r"\x89ÛÏ", "", text)
 text = re.sub(r"China\x89Ûªs", "China's", text)
 text = re.sub(r"let\x89Ûªs", "let's", text)
 text = re.sub(r"\x89Û÷", "", text)
 text = re.sub(r"\x89Ûª", "", text)
 text = re.sub(r"\x89Û\x9d", "", text)
 text = re.sub(r"å_", "", text)
 text = re.sub(r"\x89Û¢", "", text)
 text = re.sub(r"\x89Û¢åÊ", "", text)
 text = re.sub(r"fromåÊwounds", "from wounds", text)
 text = re.sub(r"åÊ", "", text)
 text = re.sub(r"åÈ", "", text)
 text = re.sub(r"JapÌ_n", "Japan", text)
 text = re.sub(r"Ì©", "e", text)
 text = re.sub(r"å¨", "", text)
 text = re.sub(r"SuruÌ¤", "Suruc", text)
 text = re.sub(r"åÇ", "", text)
 text = re.sub(r"å£3million", "3 million", text)
 text = re.sub(r"åÀ", "", text)
 text = re.sub(r'https?://\S+|www\.\S+', '', text)
 text = re.sub(r'<.*?>', '', text)
 text = re.sub(r'[\d]+', ' ', text)
 return text<feature_engineering>
dft2 = pd.concat([dft2['PassengerId'], pd.DataFrame(y_pred, columns = ['Survived'] ).astype(int)], axis=1 )
unique_targets = data.groupby('text' ).agg(unique_target=('target', pd.Series.nunique))
controversial_tweets = unique_targets[unique_targets['unique_target'] > 1].index
data = data[~data['text'].isin(controversial_tweets)]
data = data.drop_duplicates(subset='text', keep='first')
 data['text'] = data['text'].apply(clean_text)
 test['text'] = test['text'].apply(clean_text )<load_pretrained>
dft2.to_csv('titanic_results.csv', index=False )