Kaggle competition: AI Academy Intermediate Class Competition 1
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )<data_type_conversions>
TRAIN_PATH = os.path.join(".. /input/ai-academy-intermediate-class-competition-1", "BBC News Train.csv")
 df = pd.read_csv(TRAIN_PATH )
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )<create_dataframe>
df['category_id'] = df['Category'].factorize() [0]
df['category_id'][0:10]
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'欠損比率' :all_data_na})
 missing_data.head()<data_type_conversions>
category_id_df = df[['Category', 'category_id']].drop_duplicates().sort_values('category_id' )
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )<categorify>
category_to_id = dict(category_id_df.values)
 id_to_category = dict(category_id_df[['category_id', 'Category']].values )
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))<feature_engineering>
df.groupby('Category' ).category_id.count()
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']<feature_engineering>
tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', encoding='latin-1', ngram_range=(1, 2), stop_words='english')
 features = tfidf.fit_transform(df.Text ).toarray()
labels = df.category_id
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
<categorify>
sorted(category_to_id.items() )
all_data = pd.get_dummies(all_data)
 print(all_data.shape )<split>
N = 3
for Category, category_id in sorted(category_to_id.items()):
features_chi2 = chi2(features, labels == category_id)
 indices = np.argsort(features_chi2[0])
 feature_names = np.array(tfidf.get_feature_names())[indices]
unigrams = [v for v in feature_names if len(v.split(' ')) == 1]
bigrams = [v for v in feature_names if len(v.split(' ')) == 2]
print("
print(".Most correlated unigrams: 
.{}".format(' 
.'.join(unigrams[-N:])))
 print(".Most correlated bigrams: 
.{}".format(' 
.'.join(bigrams[-N:])) )
train = all_data[:ntrain]
test = all_data[ntrain:]<import_modules>
SAMPLE_SIZE = int(len(features)* 0.3)
 np.random.seed(0)
 indices = np.random.choice(range(len(features)) , size=SAMPLE_SIZE, replace=False)
 projected_features = TSNE(n_components=2, random_state=0 ).fit_transform(features[indices])
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb<compute_train_metric>
my_id = 0
projected_features[(labels[indices] == my_id ).values]
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<choose_model_class>
models = [
RandomForestClassifier(n_estimators=200, max_depth=3, random_state=0),
MultinomialNB() ,
LogisticRegression(random_state=0),
]
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))<choose_model_class>
CV = 5
cv_df = pd.DataFrame(index=range(CV * len(models)))
 entries = []
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))<choose_model_class>
for model in models:
model_name = model.__class__.__name__
accuracies = cross_val_score(model, features, labels, scoring='accuracy', cv=CV)
 for fold_idx, accuracy in enumerate(accuracies):
entries.append(( model_name, fold_idx, accuracy))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )<choose_model_class>
cv_df = pd.DataFrame(entries, columns=['model_name', 'fold_idx', 'accuracy'] )
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )<choose_model_class>
cv_df.groupby('model_name' ).accuracy.mean()
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1 )<choose_model_class>
model = LogisticRegression(random_state=0)
 X_train, X_test, y_train, y_test, indices_train, indices_test = train_test_split(features, labels, df.index, test_size=0.33, random_state=0)
 model.fit(X_train, y_train)
 y_pred_proba = model.predict_proba(X_test)
 y_pred = model.predict(X_test )
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )<compute_test_metric>
for predicted in category_id_df.category_id:
for actual in category_id_df.category_id:
if predicted != actual and conf_mat[actual, predicted] >= 2:
print("'{}' predicted as '{}' : {} examples.".format(id_to_category[actual], id_to_category[predicted], conf_mat[actual, predicted]))
display(df.loc[indices_test[(y_test == actual)&(y_pred == predicted)]]['Text'])
 print('' )
score = rmsle_cv(lasso)
 print("Lasso score: {:.4f}({:.4f})".format(score.mean() , score.std()))<compute_test_metric>
model.fit(features, labels )
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})".format(score.mean() , score.std()))<compute_test_metric>
N = 5
for Category, category_id in sorted(category_to_id.items()):
indices = np.argsort(model.coef_[category_id])
 feature_names = np.array(tfidf.get_feature_names())[indices]
unigrams = [v for v in reversed(feature_names)if len(v.split(' ')) == 1][:N]
bigrams = [v for v in reversed(feature_names)if len(v.split(' ')) == 2][:N]
print("
print(".Top unigrams: 
.{}".format(' 
.'.join(unigrams)))
 print(".Top bigrams: 
.{}".format(' 
.'.join(bigrams)) )
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})".format(score.mean() , score.std()))<compute_test_metric>
texts = ["Hooli stock price soared after a dip in PiedPiper revenue growth.",
"Captain Tsubasa scores a magnificent goal for the Japanese team.",
"Merryweather mercenaries are sent on another mission, as government oversight groups call for new sanctions.",
"Beyoncé releases a new album, tops the charts in all of south-east Asia!",
"You won't guess what the latest trend in data analysis is!"]
text_features = tfidf.transform(texts)
 predictions = model.predict(text_features)
 for text, predicted in zip(texts, predictions):
print('"{}"'.format(text))
print(" - Predicted as: '{}'".format(id_to_category[predicted]))
print("" )
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})".format(score.mean() , score.std()))<compute_test_metric>
TEST_PATH = os.path.join(".. /input/bbc-news-test", "BBC News Test.csv")
 test_df = pd.read_csv(TEST_PATH)
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})".format(score.mean() , score.std()))<compute_test_metric>
test_features = tfidf.transform(test_df.Text.tolist())
 Y_pred = model.predict(test_features)
 Y_pred
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})".format(score.mean() , score.std()))<train_model>
Y_pred_name =[]
for cat_id in Y_pred :
Y_pred_name.append(id_to_category[cat_id] )
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<compute_train_metric>
submission = pd.DataFrame({
"ArticleId": test_df["ArticleId"],
"Category": Y_pred_name
} )
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})".format(score.mean() , score.std()))<train_model>
submission.to_csv('submission.csv', index=False )