Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<feature_engineering>
train_data = pd.read_csv(".. /input/train.csv")
 test_data = pd.read_csv(".. /input/test.csv" )
train_feature = add_external_features(train_df, train_feature)
 test_feature = add_external_features(test_df, test_feature )<string_transform>
gle = LabelEncoder()
train_color_labels = gle.fit_transform(train_data['color'])
 test_color_labels = gle.fit_transform(test_data['color'])
 train_data['color_labels'] = train_color_labels
test_data['color_labels'] = test_color_labels
stop_words = nltk.corpus.stopwords.words('english')
 symbol = ["'", '"', ':', ';', '.', ',', '-', '!', '?', "'s", ")", "(", "...", '``', "''", "/", "$", "%", "*", "&", "{", "}", "[", "]"]
def get_prevalent(texts, top_count=15):
tokenized_sents = [nltk.word_tokenize(i)for i in texts]
tokenized_sents = [flatten for inner in tokenized_sents for flatten in inner]
fdist = nltk.FreqDist(w.lower() for w in tokenized_sents if w.lower() not in stop_words + symbol)
 comments = fdist.most_common(top_count)
 return [word[0] for word in comments]
for column in input_columns:
words = get_prevalent(train_df[column])
 print(column, words)
 for word in words:
for df, feature in [[train_df, train_feature], [test_df, test_feature]]:
feature[column+'_num'+word] = df[column].apply(lambda comment: comment.count(word))<feature_engineering>
Y = train_data['type']
train_data = train_data.drop(['type', 'id', 'color'], axis=1)
 test_data = test_data.drop(['id', 'color'], axis=1 )
find = re.compile(r"^[^.]*")
 train_df['netloc'] = train_df['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 test_df['netloc'] = test_df['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 count_columns = ['question_title', 'question_user_name', 'answer_user_name', 'category', 'netloc']
for col in count_columns:
value = train_df[col].value_counts()
train_feature[col+'_count'] = train_df[col].map(value)
 test_feature[col+'_count'] = test_df[col].map(value ).fillna(1)
 for col in train_feature.columns:
train_mean = np.nanmean(train_feature[col].values)
 train_feature[col].fillna(train_mean, inplace=True)
 test_feature[col].fillna(train_mean, inplace=True)
 print("train: nan=", np.sum(np.sum(pd.isnull(train_feature))))
print("test : nan=", np.sum(np.sum(pd.isnull(test_feature))))
scaler = MinMaxScaler()
scaler.fit(train_feature)
 train_feature = pd.DataFrame(scaler.transform(train_feature), columns=train_feature.columns)
 test_feature = pd.DataFrame(scaler.transform(test_feature), columns=test_feature.columns)
 del scaler
gc.collect()<categorify>
train_data =(train_data - train_data.mean())/ train_data.std()
test_data =(test_data - test_data.mean())/ test_data.std()
def convert_label(df, result_df, column, count=10):
labels = [(df[column].values>=(rate/count)).astype(np.int)for rate in range(1, count+1)]
columns = ['{}_{}'.format(column, i)for i in range(count)]
labels = np.array(labels ).T
label_df = pd.DataFrame(labels, columns=columns)
 result_df = pd.concat(( result_df, label_df), axis=1)
 return result_df
def convert_label_origin(df, result_df, column, count=10):
columns = ['{}_{}'.format(column, i)for i in range(count)]
labels = df[columns].values
values = []
for i in range(len(labels)) :
value = 0.0
for j in range(count):
if labels[i][j] > 0.5:
value =(j+1)/ count
values.append(value)
 label_df = pd.DataFrame(values, columns=[column])
 result_df = pd.concat(( result_df, label_df), axis=1)
 return result_df<create_dataframe>
model_to_set = OneVsRestClassifier(LogisticRegression(penalty = 'l2', max_iter = 1000))
parameters = {
"estimator__C": [0.1, 0.5, 0.7, 1.0, 1.2, 2, 5, 10, 20, 100],
"estimator__solver": ['newton-cg', 'lbfgs', 'sag'],
"estimator__multi_class" : ['multinomial', 'ovr']
}
model_tunning = GridSearchCV(model_to_set, param_grid=parameters,
cv = 2)
 model_tunning.fit(train_data, Y)
 print(model_tunning.best_score_)
 print(model_tunning.best_params_ )
label_convert_count = 12
train_feature2 = pd.DataFrame()
test_feature2 = pd.DataFrame()
for column in train_feature.columns:
train_feature2 = convert_label(train_feature, train_feature2, column, label_convert_count)
 test_feature2 = convert_label(test_feature, test_feature2, column, label_convert_count)
 train_feature = train_feature2.copy()
test_feature = test_feature2.copy()
del train_feature2, test_feature2<feature_engineering>
ovr = OneVsRestClassifier(LogisticRegression(penalty = "l2", C = 0.1, multi_class = 'ovr', solver = 'newton-cg'))
ovr_fitted = ovr.fit(train_data, Y)
 res = ovr_fitted.predict(test_data )
<categorify><EOS> 
sample_data = pd.read_csv(".. /input/sample_submission.csv")
 sample_data['type'] = res
sample_data.to_csv('prediction.csv', index = False )