Kaggle competition: AI Academy Intermediate Class Competition 1
<SOS> metric: categorizationaccuracy Kaggle data source: ai-academy-intermediate-class-competition-1<data_type_conversions>
name = os.path.join(".. /input/ai-academy-intermediate-class-competition-1", "BBC News Train.csv")
 data = pd.read_csv(name)
 data = data[["Text", "Category"]]
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )<groupby>
vectorizer = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', encoding='latin-1', ngram_range=(1, 2), stop_words='english')
 X = vectorizer.fit_transform(data["Text"])
 print(len(vectorizer.get_feature_names()))
print(X.shape )
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))<data_type_conversions>
data["category_id"]=data["Category"].factorize() [0]
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )<data_type_conversions>
data = data[["Text", "category_id", "Category"]]
data
category_id_data = data[['Category', 'category_id']].drop_duplicates().sort_values('category_id')
 category_to_id = dict(category_id_data.values)
 id_to_category = dict(category_id_data[['category_id', 'Category']].values)
 category_id_data
id_to_category
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )<data_type_conversions>
tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', encoding='latin-1', ngram_range=(1, 2), stop_words='english')
 features = tfidf.fit_transform(data.Text ).toarray()
print(features)
 labels = data.category_id
print(labels)
 features.shape
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )<data_type_conversions>
N = 5
for category, category_id in sorted(category_to_id.items()):
features_chi2 = chi2(features, labels == category_id)
 indices = np.argsort(features_chi2[0])
 feature_names = np.array(tfidf.get_feature_names())[indices]
unigrams = [v for v in feature_names if len(v.split(' ')) == 1]
bigrams = [v for v in feature_names if len(v.split(' ')) == 2]
print("
print(".Most correlated unigrams: 
.{}".format(' 
.'.join(unigrams[-N:])))
 print(".Most correlated bigrams: 
.{}".format(' 
.'.join(bigrams[-N:])) )
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )<data_type_conversions>
X_train, X_test, y_train, y_test, indices_train, indices_test = train_test_split(features, labels, data.index, test_size=0.33, random_state=0)
 for model in models:
model.fit(X_train, y_train)
 y_pred_proba = model.predict_proba(X_test)
 y_pred = model.predict(X_test)
 accuracy = accuracy_score(y_test, y_pred)
 print(accuracy )
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )<data_type_conversions>
model = models[2]
model.fit(features, labels)
 model.coef_
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )<drop_column>
test_data = pd.read_csv(".. /input/bbc-test/BBC News Test.csv")
 test_data
all_data = all_data.drop(['Utilities'], axis=1 )<data_type_conversions>
test_data.Text.tolist()
test_features = tfidf.transform(test_data.Text.tolist())
 Y_pred = model.predict(test_features)
 Y_pred
submission = []
for pred in Y_pred:
submission.append(id_to_category[pred])
 submission
all_data["Functional"] = all_data["Functional"].fillna("Typ" )<data_type_conversions>
submission = pd.DataFrame({
"ArticleId": test_data["ArticleId"],
"Category": submission
})
 submission
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )<data_type_conversions>
submission.to_csv('submission.csv', index=False )
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )<categorify>
submission.to_csv('submission.csv', index=False )