Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
eng_stopwords = set(stopwords.words("english"))<feature_engineering>
y = df["type"]
indexes_test = df_test["id"]
df = df.drop(["type","color","id"],axis=1)
 df_test = df_test.drop(["color","id"],axis=1 )
for df, feature in [[train_df, train_feature], [test_df, test_feature]]:
for column in input_columns:
feature[column+'_total_length'] = df[column].apply(len)
 feature[column+'_capitals'] = df[column].apply(lambda comment: sum(1 for c in comment if c.isupper()))
feature[column+'_caps_vs_length'] = feature.apply(lambda row: float(row[column+'_capitals'])/float(row[column+'_total_length']),axis=1)
 feature[column+'_num_exclamation_marks'] = df[column].apply(lambda comment: comment.count('!'))
feature[column+'_num_question_marks'] = df[column].apply(lambda comment: comment.count('?'))
feature[column+'_num_punctuation'] = df[column].apply(lambda comment: sum(comment.count(w)for w in '.,;:'))
feature[column+'_num_symbols'] = df[column].apply(lambda comment: sum(comment.count(w)for w in '*&$%'))
feature[column+"_num_chars"] = df[column].apply(lambda x: len(str(x)))
 feature[column+'_num_words'] = df[column].apply(lambda comment: len(comment.split()))
feature[column+'_num_unique_words'] = df[column].apply(lambda comment: len(set(w for w in comment.split())))
 feature[column+'_words_vs_unique'] = feature[column+'_num_unique_words'] / feature[column+'_num_words']
feature[column+'_num_smilies'] = df[column].apply(lambda comment: sum(comment.count(w)for w in(':-)', ':)', ';-)', ';)')))
 feature[column+"_num_stopwords"] = df[column].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
feature[column+"_num_punctuations"] =df[column].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
feature[column+"_num_words_upper"] = df[column].apply(lambda x: len([w for w in str(x ).split() if w.isupper() ]))<categorify>
df = pd.get_dummies(df)
 df_test = pd.get_dummies(df_test )
nlp = English()
sentencizer = nlp.create_pipe('sentencizer')
 nlp.add_pipe(sentencizer)
 ans_user_and_category=train_df[train_df[['answer_user_name', 'category']].duplicated() ][['answer_user_name', 'category']].values
ans_user_and_category.shape<feature_engineering>
X_train, X_test, y_train, y_test = train_test_split(df, y, test_size=0.3, random_state=0 )
def add_question_metadata_features(text):
doc=nlp(text)
 indirect = 0
choice_words=0
reason_explanation_words = 0
question_count = 0
for sent in doc.sents:
if '?' in sent.text and '?' == sent.text[-1]:
question_count += 1
for token in sent:
if token.text.lower() =='why':
reason_explanation_words+=1
elif token.text.lower() =='or':
choice_words+=1
if question_count==0:
indirect+=1
return np.array([indirect, question_count, reason_explanation_words, choice_words])
 def question_answer_author_same(df):
q_username = df['question_user_name']
a_username = df['answer_user_name']
author_same=[]
for i in range(len(df)) :
if q_username[i] == a_username[i]:
author_same.append(int(1))
else:
author_same.append(int(0))
return author_same
def add_external_features(df, feature):
feature['question_vs_answer_length'] = feature['question_body_num_words']/feature['answer_num_words']
feature['q_a_author_same'] = question_answer_author_same(df)
 answer_user_cat = []
for i in df[['answer_user_name', 'category']].values:
if i in ans_user_and_category:
answer_user_cat.append(int(1))
else:
answer_user_cat.append(int(0))
feature['answer_user_cat'] = answer_user_cat
handmade_features=[]
for text in df['question_body'].values:
handmade_features.append(add_question_metadata_features(text))
feature = pd.concat([ feature, pd.DataFrame(handmade_features, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])], axis=1)
 return feature<feature_engineering>
lr = LogisticRegression(penalty='l2',C=1000000)
 lr.fit(X_train,y_train)
 y_pred= lr.predict(X_test)
 print(classification_report(y_pred,y_test))
<string_transform><EOS> 
y_pred = lr.predict(df_test)
 Y = pd.DataFrame()
Y["id"] = indexes_test
Y["type"] = y_pred
Y.to_csv("submission.csv",index=False )