Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
def prediction_v7(test_dataloader, model, classifier, shape, batch_size=8):
print("Running Prediction...")
 t0 = time.time()
model.eval()
classifier.eval()
test_preds = np.zeros(( shape, 30))
for step, batch in enumerate(test_dataloader):
batch = tuple(t.to(device)for t in batch)
 test_input_ids, test_input_mask = batch
with torch.no_grad() :
attention_mask =(test_input_ids > 0)
 last_layer, _ , hidden_layers= model(input_ids=test_input_ids,
token_type_ids=test_input_mask,
attention_mask=attention_mask)
 cls = torch.cat(tuple(hidden_layers[i][:, 0, :] for i in [-1, -2, -3, -4]), dim = -1)
 prediction = classifier(cls)
 test_preds[step * batch_size:(step + 1)* batch_size] = prediction.detach().cpu().numpy()
print(" Prediction took: {:}".format(format_time(time.time() - t0)))
 return test_preds<predict_on_test>
train_data=pd.read_csv('/kaggle/input/ghouls-goblins-and-ghosts-boo/train.csv.zip')
 train_data.head()
def prediction(test_dataloader, model, classifier, shape, batch_size=8):
print("Running Prediction...")
 t0 = time.time()
model.eval()
classifier.eval()
test_preds = np.zeros(( shape, 30))
for step, batch in enumerate(test_dataloader):
batch = tuple(t.to(device)for t in batch)
 test_input_ids, test_input_mask= batch
with torch.no_grad() :
attention_mask =(test_input_ids > 0)
 last_layer, _ , hidden_layers= model(input_ids=test_input_ids,
token_type_ids=test_input_mask,
attention_mask=attention_mask)
 outputs = torch.cat(tuple(hidden_layers[i] for i in [-1, -2, -3, -4]), dim = -1)
 pooling_layer = torch.nn.AdaptiveAvgPool2d(( 1, 3072))
cls = pooling_layer(outputs ).squeeze()
prediction = classifier(cls)
 test_preds[step * batch_size:(step + 1)* batch_size] = prediction.detach().cpu().numpy()
print(" Prediction took: {:}".format(format_time(time.time() - t0)))
 return test_preds<create_dataframe>
train_data['type'].value_counts()
ds_test = QuestDataset(df_test, labeled=False)
 test_dataloader = torch.utils.data.DataLoader(ds_test, **params )<load_pretrained>
test_data=pd.read_csv('/kaggle/input/ghouls-goblins-and-ghosts-boo/test.csv.zip')
 test_data.head()
config = config = BertConfig.from_pretrained(configPath)
 config.output_hidden_states = True<define_variables>
train_data['color'].value_counts()
test_predictions = []<load_pretrained>
test_data['color'].value_counts()
for i in range(5):
model_path = f'.. /input/google-quest-pytorch-v7/bert-base-{i}.h5py'
classifier_path = f'.. /input/google-quest-pytorch-v7/bert-base-classifier-{i}.h5py'
bert_model_weights = torch.load(model_path)
 model = BertModel.from_pretrained(configPath, state_dict = bert_model_weights, config=config)
 classifier.load_state_dict(torch.load(classifier_path))
model.cuda()
classifier.cuda()
test_predictions.append(prediction_v7(test_dataloader, model, classifier, len(df_test)))
 print('Fold Done')
 print(' ' )<prepare_output>
train_data=pd.concat([train_data,pd.get_dummies(train_data['color'])],axis=1)
 train_data.drop('color',axis=1,inplace=True)
 train_data.head()
final_predictions = np.mean(test_predictions, axis=0 )<drop_column>
test_data=pd.concat([test_data,pd.get_dummies(test_data['color'])],axis=1)
 test_data.drop('color',axis=1,inplace=True)
 test_data.head()
del test_predictions, ds_test, df_test, test_dataloader, model, classifier; gc.collect()<set_options>
X=train_data.drop(['id','type'],axis=1)
 y=pd.get_dummies(train_data['type'] )
torch.cuda.empty_cache()<set_options>
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.1,random_state=42)
 print(X_train.shape,y_train.shape)
 print(X_test.shape,y_test.shape )
InteractiveShell.ast_node_interactivity = "all"
warnings.filterwarnings('ignore')
<set_options>
from tensorflow import keras
from keras.layers import Dense,Dropout
from keras.models import Sequential
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
model=Sequential()
model.add(Dense(100,input_shape=(X.shape[1],)))
 model.add(Dense(100,activation='relu'))
model.add(Dense(100,activation='relu'))
model.add(Dense(3,activation='softmax'))
model.summary()
SEED = 1414
DATA_DIR = ".. /input/google-quest-challenge/"
WORK_DIR = ".. /working/"
BERT_VOCAB_PATH = ".. /input/roberta-large/roberta-large-vocab.json"
BERT_MERGES_PATH = ".. /input/roberta-large/roberta-large-merges.txt"
BERT_MODEL_PATH = ".. /input/roberta-large/roberta-large-pytorch_model.bin"
BERT_CONFIG_PATH = ".. /input/roberta-large/roberta-large-config.json"
input_columns = ['question_title', 'question_body', 'answer']
seed_everything(SEED)
 batch_size = 32<load_from_csv>
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'] )
test_df = pd.read_csv(os.path.join(DATA_DIR,"test.csv"))<feature_engineering>
train=model.fit(x=X_train,y=y_train,batch_size=16,epochs=12,verbose=2,validation_data=(X_test,y_test))
eng_stopwords = set(stopwords.words("english"))
def add_external_features(df):
eng_stopwords = set(stopwords.words("english"))
df['question_body'] = df['question_body'].apply(lambda x:str(x))
df['question_num_words'] = df.question_body.str.count('\S+')
 df['question_title_num_words'] = df.question_title.str.count('\S+')
 df['answer'] = df['answer'].apply(lambda x:str(x))
df['answer_num_words'] = df.answer.str.count('\S+')
 df["question_title_num_unique_words"] = df["question_title"].apply(lambda x: len(set(str(x ).split())))
 df["question_body_num_unique_words"] = df["question_body"].apply(lambda x: len(set(str(x ).split())))
 df["answer_num_unique_words"] = df["answer"].apply(lambda x: len(set(str(x ).split())))
 df["question_title_num_chars"] = df["question_title"].apply(lambda x: len(str(x)))
 df["question_body_num_chars"] = df["question_body"].apply(lambda x: len(str(x)))
 df["answer_num_chars"] = df["answer"].apply(lambda x: len(str(x)))
 df['qt_words'] = df['question_title'].apply(lambda s: [f for f in s.split() if f not in eng_stopwords])
 df['q_words'] = df['question_body'].apply(lambda s: [f for f in s.split() if f not in eng_stopwords])
 df['a_words'] = df['answer'].apply(lambda s: [f for f in s.split() if f not in eng_stopwords])
 df['qa_word_overlap'] = df.apply(lambda s: len(np.intersect1d(s['q_words'], s['a_words'])) , axis = 1)
 df['qt_word_overlap'] = df.apply(lambda s: len(np.intersect1d(s['qt_words'], s['a_words'])) , axis = 1)
 df['qa_word_overlap_norm'] = df.apply(lambda s: s['qa_word_overlap']/(len(s['a_words'])+ len(s['q_words'])- s['qa_word_overlap']), axis = 1)
 df['qta_word_overlap_norm'] = df.apply(lambda s: s['qt_word_overlap']/(len(s['a_words'])+ len(s['qt_words'])- s['qt_word_overlap']), axis = 1)
 df.drop(['q_words', 'a_words', 'qt_words'], axis = 1, inplace = True)
 return df<feature_engineering>
import matplotlib.pyplot as plt
test_df = add_external_features(test_df )<normalization>
pred=model.predict(test_data.drop('id',axis=1))
pred_final=[np.argmax(i)for i in pred]
submission = pd.DataFrame({'id':test_data['id'], 'type':pred_final})
 submission.head()
handmade_cols = ["question_body_num_unique_words", 'question_num_words',
"question_title_num_unique_words", "question_title_num_words",
"answer_num_unique_words", "answer_num_words"]
with open('/kaggle/input/d-large-roberta-b32-steplr3-drop-tb-rmf-h56-2/C_scaler.pickle', mode='rb')as f:
num_words_scaler = pickle.load(f)
 test_df[handmade_cols]= num_words_scaler.transform(test_df[handmade_cols].values)
 train_handmade_features= test_df[handmade_cols + ['qa_word_overlap_norm', 'qta_word_overlap_norm']].values<categorify>
submission['type'].replace(to_replace=[0,1,2],value=['Ghost','Ghoul','Goblin'],inplace=True)
 submission.head()
def label_encoder(x, dict_reverse):
try:
return dict_reverse[x]
except:
return 0
with open('/kaggle/input/d-large-roberta-b32-steplr3-drop-tb-rmf-h56-2/C_category.pickle', mode='rb')as f:
category_dict_reverse = pickle.load(f)
 test_df['category'] = test_df['category'].apply(lambda x: label_encoder(x, category_dict_reverse))
with open('/kaggle/input/d-large-roberta-b32-steplr3-drop-tb-rmf-h56-2/C_host.pickle', mode='rb')as f:
host_dict_reverse = pickle.load(f)
 test_df['host'] = test_df['host'].apply(lambda x: label_encoder(x, host_dict_reverse))
n_cat = len(category_dict_reverse)+ 1
cat_emb = 256
n_host = len(host_dict_reverse)+ 1
host_emb = 256<load_pretrained>
submission.to_csv('.. /working/submission.csv', index=False )