Kaggle competition: AI Academy Intermediate Class Competition 1
train_df.head()<feature_engineering>
TRAIN_PATH = os.path.join(".. /input/ai-academy-intermediate-class-competition-1", "BBC News Train.csv")
 df = pd.read_csv(TRAIN_PATH )
<correct_missing_values>
df['category_id'] = df['Category'].factorize() [0]
df['category_id'][0:10]
freq_port = train_df.Embarked.dropna().mode() [0]
freq_port<sort_values>
category_id_df = df[['Category', 'category_id']].drop_duplicates().sort_values('category_id' )
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<data_type_conversions>
category_to_id = dict(category_id_df.values)
 id_to_category = dict(category_id_df[['category_id', 'Category']].values )
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()<data_type_conversions>
df.groupby('Category' ).category_id.count()
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 test_df.head()<categorify>
tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', encoding='latin-1', ngram_range=(1, 2), stop_words='english')
 features = tfidf.fit_transform(df.Text ).toarray()
labels = df.category_id
for dataset in combine:
dataset.loc[ dataset['Fare'] <= 128, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 128)&(dataset['Fare'] <= 256.2), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 265.2)&(dataset['Fare'] <= 384.3), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 384.3, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 train_df = train_df.drop(['FareBand'], axis=1)
 combine = [train_df, test_df]
train_df.head(10 )<drop_column>
sorted(category_to_id.items() )
train_df = train_df.drop(['Embarked', 'Title'], axis=1)
 test_df = test_df.drop(['Embarked', 'Title'], axis=1)
 combine = [train_df, test_df]
train_df.head()<drop_column>
N = 3
for Category, category_id in sorted(category_to_id.items()):
features_chi2 = chi2(features, labels == category_id)
 indices = np.argsort(features_chi2[0])
 feature_names = np.array(tfidf.get_feature_names())[indices]
unigrams = [v for v in feature_names if len(v.split(' ')) == 1]
bigrams = [v for v in feature_names if len(v.split(' ')) == 2]
print("
print(".Most correlated unigrams: 
.{}".format(' 
.'.join(unigrams[-N:])))
 print(".Most correlated bigrams: 
.{}".format(' 
.'.join(bigrams[-N:])) )
train_df = train_df.drop(['Pclass'], axis=1)
 test_df = test_df.drop(['Pclass'], axis=1)
 combine = [train_df, test_df]
train_df.head()<prepare_x_and_y>
SAMPLE_SIZE = int(len(features)* 0.3)
 np.random.seed(0)
 indices = np.random.choice(range(len(features)) , size=SAMPLE_SIZE, replace=False)
 projected_features = TSNE(n_components=2, random_state=0 ).fit_transform(features[indices])
X_train = train_df.drop("Survived", axis=1)
 Y_train = train_df["Survived"]
X_test = test_df.drop("PassengerId", axis=1 ).copy()
X_train.shape, Y_train.shape, X_test.shape<choose_model_class>
my_id = 0
projected_features[(labels[indices] == my_id ).values]
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2)
 acc_log<compute_train_metric>
models = [
RandomForestClassifier(n_estimators=200, max_depth=3, random_state=0),
MultinomialNB() ,
LogisticRegression(random_state=0),
]
svc = SVC()
svc.fit(X_train, Y_train)
 Y_pred = svc.predict(X_test)
 acc_svc = round(svc.score(X_train, Y_train)* 100, 2)
 acc_svc<predict_on_test>
CV = 5
cv_df = pd.DataFrame(index=range(CV * len(models)))
 entries = []
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2)
 acc_knn<predict_on_test>
for model in models:
model_name = model.__class__.__name__
accuracies = cross_val_score(model, features, labels, scoring='accuracy', cv=CV)
 for fold_idx, accuracy in enumerate(accuracies):
entries.append(( model_name, fold_idx, accuracy))
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred = gaussian.predict(X_test)
 acc_gaussian = round(gaussian.score(X_train, Y_train)* 100, 2)
 acc_gaussian<compute_train_metric>
BBC Ncv_df = pd.DataFrame(entries, columns=['model_name', 'fold_idx', 'accuracy'] )
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
 Y_pred = perceptron.predict(X_test)
 acc_perceptron = round(perceptron.score(X_train, Y_train)* 100, 2)
 acc_perceptron<compute_train_metric>
cv_df.groupby('model_name' ).accuracy.mean()
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 Y_pred = linear_svc.predict(X_test)
 acc_linear_svc = round(linear_svc.score(X_train, Y_train)* 100, 2)
 acc_linear_svc<compute_train_metric>
model = LogisticRegression(random_state=0)
 X_train, X_test, y_train, y_test, indices_train, indices_test = train_test_split(features, labels, df.index, test_size=0.33, random_state=0)
 model.fit(X_train, y_train)
 y_pred_proba = model.predict_proba(X_test)
 y_pred = model.predict(X_test )
sgd = SGDClassifier()
sgd.fit(X_train, Y_train)
 Y_pred = sgd.predict(X_test)
 acc_sgd = round(sgd.score(X_train, Y_train)* 100, 2)
 acc_sgd<predict_on_test>
for predicted in category_id_df.category_id:
for actual in category_id_df.category_id:
if predicted != actual and conf_mat[actual, predicted] >= 2:
print("'{}' predicted as '{}' : {} examples.".format(id_to_category[actual], id_to_category[predicted], conf_mat[actual, predicted]))
display(df.loc[indices_test[(y_test == actual)&(y_pred == predicted)]]['Text'])
 print('' )
decision_tree = DecisionTreeClassifier(random_state = 0, criterion="gini", max_depth = 13)
 decision_tree.fit(X_train, Y_train)
 Y_pred = decision_tree.predict(X_test)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2)
 acc_decision_tree<choose_model_class>
model.fit(features, labels )
random_forest = RandomForestClassifier(criterion='gini',
n_estimators=500,
max_depth=10,
min_samples_split=5,
min_samples_leaf=1,
max_features='auto',
oob_score=True,
random_state=2,
n_jobs=-1)
 random_forest.fit(X_train, Y_train)
 Y_pred = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 acc_random_forest 
<predict_on_test>
N = 5
for Category, category_id in sorted(category_to_id.items()):
indices = np.argsort(model.coef_[category_id])
 feature_names = np.array(tfidf.get_feature_names())[indices]
unigrams = [v for v in reversed(feature_names)if len(v.split(' ')) == 1][:N]
bigrams = [v for v in reversed(feature_names)if len(v.split(' ')) == 2][:N]
print("
print(".Top unigrams: 
.{}".format(' 
.'.join(unigrams)))
 print(".Top bigrams: 
.{}".format(' 
.'.join(bigrams)) )
XGBClassifier
xg = XGBClassifier(n_estimators=300, max_depth=13, random_state=2)
 xg.fit(X_train, Y_train)
 Y_pred_xg = xg.predict(X_test)
 xg.score(X_train, Y_train)
 acc_xg = round(xg.score(X_train, Y_train)* 100, 2)
 acc_xg<create_dataframe>
texts = ["Hooli stock price soared after a dip in PiedPiper revenue growth.",
"Captain Tsubasa scores a magnificent goal for the Japanese team.",
"Merryweather mercenaries are sent on another mission, as government oversight groups call for new sanctions.",
"Beyoncé releases a new album, tops the charts in all of south-east Asia!",
"You won't guess what the latest trend in data analysis is!"]
text_features = tfidf.transform(texts)
 predictions = model.predict(text_features)
 for text, predicted in zip(texts, predictions):
print('"{}"'.format(text))
print(" - Predicted as: '{}'".format(id_to_category[predicted]))
print("" )
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron',
'Stochastic Gradient Decent', 'Linear SVC',
'Decision Tree'],
'Score': [acc_svc, acc_knn, acc_log,
acc_random_forest, acc_gaussian, acc_perceptron,
acc_sgd, acc_linear_svc, acc_decision_tree]})
 models.sort_values(by='Score', ascending=False )<save_to_csv>
TEST_PATH = os.path.join(".. /input/bbc-test", "BBC News Test(1 ).csv")
 test_df = pd.read_csv(TEST_PATH)
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv('submission.csv', index=False )<import_modules>
test_features = tfidf.transform(test_df.Text.tolist())
 Y_pred = model.predict(test_features)
 Y_pred
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler<load_from_csv>
Y_pred_name =[]
for cat_id in Y_pred :
Y_pred_name.append(id_to_category[cat_id] )
train_file = ".. /input/digit-recognizer/train.csv"
test_file = ".. /input/digit-recognizer/test.csv"
sample_submission = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv' )<split>
submission = pd.DataFrame({
"ArticleId": test_df["ArticleId"],
"Category": Y_pred_name
} )
<data_type_conversions><EOS> 
submission.to_csv('submission.csv', index=False )