Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<data_type_conversions>
%matplotlib inline
for col in train_feature.columns:
train_mean = np.nanmean(train_feature[col].values)
 train_feature[col].fillna(train_mean, inplace=True)
 test_feature[col].fillna(train_mean, inplace=True)
 print("train: nan=", np.sum(np.sum(pd.isnull(train_feature))))
print("test : nan=", np.sum(np.sum(pd.isnull(test_feature)))) 
<define_variables>
testset = pd.read_csv(".. /input/test.csv")
 trainset = pd.read_csv(".. /input/train.csv" )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '\xa0', '\t',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '\u3000', '\u202f',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '«',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def clean_text(x):
x = str(x ).replace("
","")
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x<string_transform>
trainset['hair_soul'] = trainset.apply(lambda row: row['hair_length']*row['has_soul'],axis=1)
 testset['hair_soul'] = testset.apply(lambda row: row['hair_length']*row['has_soul'],axis=1 )
tokenizer = TreebankWordTokenizer()
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
def replace_typical_misspell(text):
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def clean_data(df, columns: list):
for col in columns:
df[col] = df[col].apply(lambda x: clean_text(x.lower()))
df[col] = df[col].apply(lambda x: replace_typical_misspell(x))
df[col] = df[col].apply(lambda x: handle_contractions(x))
df[col] = df[col].apply(lambda x: fix_quote(x))
return df
train_df = clean_data(train_df, input_columns)
 test_df = clean_data(test_df, input_columns)
 del tokenizer<normalization>
x = trainset[["bone_length", "rotting_flesh", "hair_length", "has_soul"]]
x_hair_soul = trainset[["bone_length", "rotting_flesh", "hair_length", "has_soul", "hair_soul"]]
x_test = testset[["bone_length", "rotting_flesh", "hair_length", "has_soul"]]
x_test_hair_soul = testset[["bone_length", "rotting_flesh", "hair_length", "has_soul", "hair_soul"]]
y = trainset[["type"]]
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet3(nn.Module):
def __init__(self, num_features, num_labels, pretrained_bert):
super(NeuralNet3, self ).__init__()
self.config = transformers.GPT2Config.from_pretrained(pretrained_bert, output_hidden_states=False)
 self.gptmodel = transformers.GPT2Model(self.config)
 self.gpt = self.gptmodel.from_pretrained(pretrained_bert,config =self.config)
 self.num_features = num_features
self.num_labels = num_labels
self.encoded_dropout = SpatialDropout(0.2)
 self.pooled_dropout = nn.Dropout(0.2)
 self.feature_linear = nn.Sequential(
 nn.Linear(self.num_features, self.num_features),
nn.ReLU(inplace=True),
nn.Dropout(0.2), 
)

dense_hidden_units = self.gpt.config.hidden_size * 3 + self.num_features
self.linear1 = nn.Linear(dense_hidden_units, dense_hidden_units)
 self.linear2 = nn.Linear(dense_hidden_units, dense_hidden_units)
 self.classifier = nn.Sequential(
 nn.Linear(dense_hidden_units, num_labels), 
)
 def forward(self, ids, masks, segments, feature):
feature_output = self.feature_linear(feature)
 outputs = self.gpt(input_ids=ids, attention_mask=masks, token_type_ids=segments)
 last_hidden_state,present = self.gpt(input_ids=ids, attention_mask=masks, token_type_ids=segments)
 last_hidden_state = self.encoded_dropout(last_hidden_state)
 avg_pool = torch.mean(last_hidden_state, 1)
 max_pool, _ = torch.max(last_hidden_state, 1)
 h_conc = torch.cat(( avg_pool, max_pool, last_hidden_state[:, -1, :], feature_output), 1)
 h_conc_linear = F.relu(self.linear1(h_conc))
hidden = h_conc + h_conc_linear
h_conc_linear = F.relu(self.linear2(hidden))
hidden = hidden + h_conc_linear
return self.classifier(hidden)
<string_transform>
parameters = {'solver': ['lbfgs'], 'max_iter': [500,1000,1500], 'alpha': 10.0 ** -np.arange(1, 7), 'hidden_layer_sizes':np.arange(5, 12), 'random_state':[0,1,2,3,4,5,6,7,8,9]}
clf_grid = GridSearchCV(neural_network.MLPClassifier() , parameters, n_jobs=-1)
 parameters = {'solver': ['lbfgs'], 'max_iter': [500,1000,1500], 'alpha': 10.0 ** -np.arange(1, 7), 'hidden_layer_sizes':np.arange(5, 12), 'random_state':[0,1,2,3,4,5,6,7,8,9]}
clf_grid_hair_soul = GridSearchCV(neural_network.MLPClassifier() , parameters, n_jobs=-1 )
def compute_input_title_question(df, tokenizer, max_sequence_length=512):
input_ids, input_masks, input_segments = [], [], []
for _, instance in df.iterrows() :
title = tokenizer.tokenize(instance.question_title)
 question = tokenizer.tokenize(instance.question_body)
 if(len(title)+len(question)+3)> max_sequence_length:
if len(title)> 30:
title = title[:30]
question_len = max_sequence_length - len(title)- 3
question = question[:question_len]
title_ids = tokenizer.convert_tokens_to_ids(title)
 question_ids = tokenizer.convert_tokens_to_ids(question)
 cls_ids = tokenizer.convert_tokens_to_ids(["[CLS]"])
 sep_ids = tokenizer.convert_tokens_to_ids(["[SEP]"])
 token_ids = title_ids + question_ids
padding = [0] *(max_sequence_length - len(token_ids))
ids = token_ids + padding
masks = [1]*len(token_ids)+ padding
len_q = len(question_ids)
 len_qf = int(len(question_ids)/2)
 len_ql = len_q-len_qf
segments = [0]*(len(title_ids)) + [1]*(len(question_ids)) + padding
input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]<string_transform>
clf_grid.fit(x,y.values.ravel())
 clf_grid_hair_soul.fit(x_hair_soul,y.values.ravel())
 print("-----------------Original Features--------------------")
 print("Best score: %0.4f" % clf_grid.best_score_)
 print("Using the following parameters:")
 print(clf_grid.best_params_)
 print("-----------------Added hair_soul feature--------------")
 print("Best score: %0.4f" % clf_grid_hair_soul.best_score_)
 print("Using the following parameters:")
 print(clf_grid.best_params_)
 print("------------------------------------------------------" )
def compute_input_title_answer(df, tokenizer, max_sequence_length=512):
input_ids, input_masks, input_segments = [], [], []
for _, instance in df.iterrows() :
title = tokenizer.tokenize(instance.question_title)
 answer = tokenizer.tokenize(instance.answer)
 if(len(title)+len(answer)+3)> max_sequence_length:
if len(title)> 30:
title = title[:30]
answer_len = max_sequence_length - len(title)- 3
answer = answer[:answer_len]
title_ids = tokenizer.convert_tokens_to_ids(title)
 answer_ids = tokenizer.convert_tokens_to_ids(answer)
 cls_ids = tokenizer.convert_tokens_to_ids(["[CLS]"])
 sep_ids = tokenizer.convert_tokens_to_ids(["[SEP]"])
 token_ids = title_ids + answer_ids
padding = [0] *(max_sequence_length - len(token_ids))
ids = token_ids + padding
masks = [1]*len(token_ids)+ padding
segments = [0]*(len(title_ids)) + [1]*(len(answer_ids)) + padding
input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]<string_transform>
clf = neural_network.MLPClassifier(alpha=0.1, hidden_layer_sizes=(6), max_iter=500, random_state=3, solver='lbfgs')
 clf.fit(x, y.values.ravel() )
def compute_input_arrays(df, tokenizer, max_sequence_length=512):
input_ids, input_masks, input_segments = [], [], []
t_max_len = 35
for _, instance in df.iterrows() :
title = tokenizer.tokenize(instance.question_title)
 question = tokenizer.tokenize(instance.question_body)
 answer = tokenizer.tokenize(instance.answer)
 if(len(title)+len(question)+len(answer)+4)> max_sequence_length:
if len(title)> t_max_len:
title = title[:t_max_len]
question_len =(max_sequence_length - len(title)- 4)// 2
question = question[:question_len]
answer_len = max_sequence_length - len(title)- len(question)- 4
answer = answer[:answer_len]
title_ids = tokenizer.convert_tokens_to_ids(title)
 question_ids = tokenizer.convert_tokens_to_ids(question)
 answer_ids = tokenizer.convert_tokens_to_ids(answer)
 cls_ids = tokenizer.convert_tokens_to_ids(["[CLS]"])
 sep_ids = tokenizer.convert_tokens_to_ids(["[SEP]"])
 token_ids = cls_ids + title_ids + sep_ids + question_ids + sep_ids + answer_ids + sep_ids
padding = [0] *(max_sequence_length - len(token_ids))
ids = token_ids + padding
masks = [1]*len(token_ids)+ padding
segments = [0]*(len(title_ids)+len(question_ids)+3)+ [1]*(len(answer_ids)+1)+ padding
input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]<import_modules>
preds = clf.predict(x_test )
<feature_engineering><EOS> 
sub = pd.DataFrame(preds)
 pd.concat([testset["id"],sub], axis=1 ).rename(columns = {0: 'type'} ).to_csv("submission_neural_net.csv", index=False )