Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '\xa0', '\t',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '\u3000', '\u202f',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '«',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def clean_text(x):
x = str(x ).replace("
","")
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x<string_transform>
train_data = pd.read_csv('.. /input/ghouls-goblins-and-ghosts-boo/train.csv.zip', compression='zip')
 test_data = pd.read_csv('.. /input/ghouls-goblins-and-ghosts-boo/test.csv.zip', compression='zip' )
tokenizer = TreebankWordTokenizer()
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
def replace_typical_misspell(text):
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def clean_data(df, columns: list):
for col in columns:
df[col] = df[col].apply(lambda x: clean_text(x.lower()))
df[col] = df[col].apply(lambda x: replace_typical_misspell(x))
df[col] = df[col].apply(lambda x: handle_contractions(x))
df[col] = df[col].apply(lambda x: fix_quote(x))
return df
train_df = clean_data(train_df, input_columns)
 test_df = clean_data(test_df, input_columns)
 del tokenizer<normalization>
X = pd.get_dummies(train_data.drop(['color','type','id'], axis = 1))
Y= train_data['type']
trainid=train_data.values[:,0]
testid=test_data.values[:,0]
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size = 0.25, random_state = 0)
 X=np.array(X_train)
 Y=np.array(y_train )
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, num_features, num_labels, pretrained_bert):
super(NeuralNet, self ).__init__()
self.bert = transformers.BertModel.from_pretrained(pretrained_bert)
 self.num_features = num_features
self.num_labels = num_labels
self.encoded_dropout = SpatialDropout(0.2)
 self.pooled_dropout = nn.Dropout(0.2)
 self.feature_linear = nn.Sequential(
 nn.Linear(self.num_features, self.num_features),
nn.ReLU(inplace=True),
nn.Dropout(0.2), 
)
 dense_hidden_units = self.bert.config.hidden_size * 3 + self.num_features
self.linear1 = nn.Linear(dense_hidden_units, dense_hidden_units)
 self.linear2 = nn.Linear(dense_hidden_units, dense_hidden_units)
 self.classifier = nn.Sequential(
 nn.Linear(dense_hidden_units, num_labels), 
)
 def forward(self, ids, masks, segments, feature):
feature_output = self.feature_linear(feature)
 sequence_output, pooled_output = self.bert(input_ids=ids, attention_mask=masks, token_type_ids=segments)
 sequence_output = self.encoded_dropout(sequence_output)
 pooled_output = self.pooled_dropout(pooled_output)
 avg_pool = torch.mean(sequence_output, 1)
 max_pool, _ = torch.max(sequence_output, 1)
 h_conc = torch.cat(( avg_pool, max_pool, pooled_output, feature_output), 1)
 h_conc_linear = F.relu(self.linear1(h_conc))
hidden = h_conc + h_conc_linear
h_conc_linear = F.relu(self.linear2(hidden))
hidden = hidden + h_conc_linear
return self.classifier(hidden )<string_transform>
model=MLPClassifier(solver='lbfgs', alpha=0.00001, hidden_layer_sizes=(5,), random_state=0)
 model.fit(X,Y)
 print(model)
 y_pred=model.predict(X_test )
def compute_input_title_question(df, tokenizer, max_sequence_length=512):
input_ids, input_masks, input_segments = [], [], []
for _, instance in df.iterrows() :
title = tokenizer.tokenize(instance.question_title)
 question = tokenizer.tokenize(instance.question_body)
 if(len(title)+len(question)+3)> max_sequence_length:
if len(title)> 30:
title = title[:30]
question_len = max_sequence_length - len(title)- 3
question = question[:question_len]
title_ids = tokenizer.convert_tokens_to_ids(title)
 question_ids = tokenizer.convert_tokens_to_ids(question)
 cls_ids = tokenizer.convert_tokens_to_ids(["[CLS]"])
 sep_ids = tokenizer.convert_tokens_to_ids(["[SEP]"])
 token_ids = cls_ids + title_ids + sep_ids + question_ids + sep_ids
padding = [0] *(max_sequence_length - len(token_ids))
ids = token_ids + padding
masks = [1]*len(token_ids)+ padding
segments = [0]*(len(title_ids)+2)+ [1]*(len(question_ids)+1)+ padding
input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]<string_transform>
y_pred=model.predict(X_test)
 model.score(X_test,y_test )
def compute_input_arrays(df, tokenizer, max_sequence_length=512):
input_ids, input_masks, input_segments = [], [], []
t_max_len = 35
for _, instance in df.iterrows() :
title = tokenizer.tokenize(instance.question_title)
 question = tokenizer.tokenize(instance.question_body)
 answer = tokenizer.tokenize(instance.answer)
 if(len(title)+len(question)+len(answer)+4)> max_sequence_length:
if len(title)> t_max_len:
title = title[:t_max_len]
question_len =(max_sequence_length - len(title)- 4)// 2
question = question[:question_len]
answer_len = max_sequence_length - len(title)- len(question)- 4
answer = answer[:answer_len]
title_ids = tokenizer.convert_tokens_to_ids(title)
 question_ids = tokenizer.convert_tokens_to_ids(question)
 answer_ids = tokenizer.convert_tokens_to_ids(answer)
 cls_ids = tokenizer.convert_tokens_to_ids(["[CLS]"])
 sep_ids = tokenizer.convert_tokens_to_ids(["[SEP]"])
 token_ids = cls_ids + title_ids + sep_ids + question_ids + sep_ids + answer_ids + sep_ids
padding = [0] *(max_sequence_length - len(token_ids))
ids = token_ids + padding
masks = [1]*len(token_ids)+ padding
segments = [0]*(len(title_ids)+len(question_ids)+3)+ [1]*(len(answer_ids)+1)+ padding
input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]<feature_engineering>
scores= cross_val_score(model, X, Y,cv=10)
 scores
scores.mean()
<feature_engineering><EOS> 
test=test_data.drop(['color','id'],axis=1)
 prediction=model.predict(test)
 submission=pd.DataFrame({'id':testid, 'type': prediction})
 submission.to_csv("submission.csv",index=False )