Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<choose_model_class>
warnings.filterwarnings("ignore")
 def clean_ColText(data, col, stem=True):

 df = data.copy()
table = str.maketrans('', '', string.punctuation)
 df[col] = df[col].map(lambda x: x.translate(table))
df[col] = df[col].map(lambda x: x.lower())
 df[col] = df[col].apply(word_tokenize)
 stop_words = set(stopwords.words('english'))
df[col] = df[col].map(lambda x: [y for y in x if not y in stop_words])
 df[col] = df[col].map(lambda x: [y for y in x if y not in ["’","’","”","“","‘","—"]])
 if stem:
porter = PorterStemmer()
df[col] = df[col].map(lambda x: [porter.stem(y)for y in x])
 return df
return df
pd.read_csv('.. /input/train.csv' ).head(10 )
models = []
model1 = EffNetB4(pretrained=False)
 models.append([model1, f".. /input/bengaliweights/effnetb4_afold4.pth"])
 models.append([model1, f".. /input/bengaliweights/effnetb4_afold5.pth"])
 models.append([model1, f".. /input/bengaliweights/effnetb4_afold6.pth"])
 TEST_BATCH_SIZE = 32
final_g_pred = []
final_v_pred = []
final_c_pred = []
final_img_ids = []
for i,v in enumerate(models):
model, path = v
model.to(DEVICE)
 model.load_state_dict(torch.load(path))
model.eval()
g_pred, v_pred, c_pred, img_ids_list = model_predict()
final_g_pred.append(g_pred)
 final_v_pred.append(v_pred)
 final_c_pred.append(c_pred)
 if i == 0:
final_img_ids.extend(img_ids_list)
<prepare_x_and_y>
sid = SentimentIntensityAnalyzer()
df = pd.read_csv('.. /input/train.csv')
 df.id = df.id.map(lambda x: int(x.replace('id','')))
 df['sent'] = df['text'].map(lambda x: sid.polarity_scores(x)['compound'])
 df = clean_ColText(df, 'text', stem=True)
 df = df.drop('id', axis=1)
 df.head(10 )
a = np.array(final_g_pred[0])
 b = np.array(final_g_pred[1] )<train_model>
df['text'] = df.text.map(lambda x: ' '.join(x))
sent = np.array(df.sent)+ 1
X = df.drop(['author','sent'], axis=1)
 y = df.author
count_vect = CountVectorizer()
tfidf_transformer = TfidfTransformer()
X = count_vect.fit_transform(X.text)
 X = tfidf_transformer.fit_transform(X)
 X = X.toarray()
print(X.shape )
0.5*a+ 0.5*b<prepare_output>
sent = sent.reshape(( sent.shape[0],1))
X = np.hstack(( X, sent))
X = sparse.csr_matrix(X)
 print(X.shape )
final_g = np.argmax(np.mean(np.array(final_g_pred),axis = 0), axis=1)
 final_v = np.argmax(np.mean(np.array(final_v_pred),axis = 0), axis=1)
 final_c = np.argmax(np.mean(np.array(final_c_pred),axis = 0), axis=1 )<define_variables>
clf = MultinomialNB(alpha=0.1)
 scores = cross_val_score(clf, X, df.author, cv=5)
 print('accuracy CV:',scores )
predictions = []
for ii, imid in enumerate(final_img_ids):
predictions.append(( f"{imid}_grapheme_root", final_g[ii]))
predictions.append(( f"{imid}_vowel_diacritic", final_v[ii]))
predictions.append(( f"{imid}_consonant_diacritic", final_c[ii]))<create_dataframe>
final = MultinomialNB(alpha=0.1)
 final.fit(X, df.author)
sub = pd.DataFrame(predictions, columns=["row_id", "target"] )<save_to_csv>
df_t = pd.read_csv('.. /input/test.csv')
 df_t['sent'] = df_t['text'].map(lambda x: sid.polarity_scores(x)['compound'])
 df_t = clean_ColText(df_t, 'text', stem=True)
 df_t['text'] = df_t.text.map(lambda x: ' '.join(x))
X_t = df_t.drop(['sent','id'], axis=1 ).text
X_t = count_vect.transform(X_t)
 X_t = tfidf_transformer.transform(X_t)
 X_t = X_t.toarray()
sent_t = np.array(df_t.sent)+ 1
sent_t = sent_t.reshape(( sent_t.shape[0],1))
print(X_t.shape, sent_t.shape)
 X_t = np.hstack(( X_t, sent_t))
X_t = sparse.csr_matrix(X_t)
 print(X_t.shape )
sub.to_csv("submission.csv", index=False )<import_modules>
preds = final.predict_proba(X_t)
 df_t['EAP'] = preds[:,0]
df_t['HPL'] = preds[:,1]
df_t['MWS'] = preds[:,2]
df_t.head()
<load_from_csv><EOS> 
df_t = df_t.drop(['text', 'sent'], axis=1)
 df_t.to_csv('results.csv', index=False )