Kaggle competition: AI Academy Intermediate Class Competition 1
df['Pclass'].value_counts()<groupby>
TRAIN_PATH = os.path.join(".. /input/ai-academy-intermediate-class-competition-1", "BBC News Train.csv")
 df = pd.read_csv(TRAIN_PATH )
grouped = df.groupby('Pclass' )<filter>
df['category_id'] = df['Category'].factorize() [0]
df['category_id'][0:10]
age60=df[(df['Age']>60)&(df['Age']<70)]<groupby>
category_id_df = df[['Category', 'category_id']].drop_duplicates().sort_values('category_id' )
age60.groupby(['Pclass','Embarked'] ).mean()<groupby>
category_to_id = dict(category_id_df.values)
 id_to_category = dict(category_id_df[['category_id', 'Category']].values )
age60.groupby(['Pclass','Embarked','SibSp','Parch'] ).mean()<feature_engineering>
df.groupby('Category' ).category_id.count()
df.loc[df['PassengerId'] == 1044, 'Fare'] = 7.9<filter>
tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', encoding='latin-1', ngram_range=(1, 2), stop_words='english')
 features = tfidf.fit_transform(df.Text ).toarray()
labels = df.category_id
df[df['Fare'].isnull() ]<filter>
sorted(category_to_id.items() )
df[df['PassengerId'] == 1044]<feature_engineering>
N = 3
for Category, category_id in sorted(category_to_id.items()):
features_chi2 = chi2(features, labels == category_id)
 indices = np.argsort(features_chi2[0])
 feature_names = np.array(tfidf.get_feature_names())[indices]
unigrams = [v for v in feature_names if len(v.split(' ')) == 1]
bigrams = [v for v in feature_names if len(v.split(' ')) == 2]
print("
print(".Most correlated unigrams: 
.{}".format(' 
.'.join(unigrams[-N:])))
 print(".Most correlated bigrams: 
.{}".format(' 
.'.join(bigrams[-N:])) )
[df.Fare]=np.round([df.Fare],1 )<string_transform>
SAMPLE_SIZE = int(len(features)* 0.3)
 np.random.seed(0)
 indices = np.random.choice(range(len(features)) , size=SAMPLE_SIZE, replace=False)
 projected_features = TSNE(n_components=2, random_state=0 ).fit_transform(features[indices])
print(df['Ticket'].str.split(expand=True))<feature_engineering>
my_id = 0
projected_features[(labels[indices] == my_id ).values]
Ticket = df['Ticket'].str.extract(' (.*)\s (.*)' )<feature_engineering>
models = [
RandomForestClassifier(n_estimators=200, max_depth=3, random_state=0),
MultinomialNB() ,
LogisticRegression(random_state=0),
]
df['Ticket_head']=Ticket[0]
df['Ticket_num']=Ticket[1]<categorify>
CV = 5
cv_df = pd.DataFrame(index=range(CV * len(models)))
 entries = []
Ticket_head = df['Ticket_head'].apply(lambda x: x.replace(".", "")if type(x)is str else float(x))
Ticket_head=Ticket_head.fillna('0' )<feature_engineering>
for model in models:
model_name = model.__class__.__name__
accuracies = cross_val_score(model, features, labels, scoring='accuracy', cv=CV)
 for fold_idx, accuracy in enumerate(accuracies):
entries.append(( model_name, fold_idx, accuracy))
df['Ticket_head'] = Ticket_head<count_values>
cv_df = pd.DataFrame(entries, columns=['model_name', 'fold_idx', 'accuracy'] )
df['Ticket_num']=df['Ticket_num'].fillna('0')
 df['Ticket_num'].value_counts()<filter>
cv_df.groupby('model_name' ).accuracy.mean()
df[df['Age'].isnull() ]<drop_column>
model = LogisticRegression(random_state=0)
 X_train, X_test, y_train, y_test, indices_train, indices_test = train_test_split(features, labels, df.index, test_size=0.33, random_state=0)
 model.fit(X_train, y_train)
 y_pred_proba = model.predict_proba(X_test)
 y_pred = model.predict(X_test )
gc = df.drop(['PassengerId'], axis=1 )<categorify>
for predicted in category_id_df.category_id:
for actual in category_id_df.category_id:
if predicted != actual and conf_mat[actual, predicted] >= 2:
print("'{}' predicted as '{}' : {} examples.".format(id_to_category[actual], id_to_category[predicted], conf_mat[actual, predicted]))
display(df.loc[indices_test[(y_test == actual)&(y_pred == predicted)]]['Text'])
 print('' )
gc['Fare']=pd.cut(gc['Fare'], 5, labels=False)
 column = 'Cabin'
gc[column] = gc[column].fillna(0)
 gc[column] = gc[column].str.extract('([A-Za-z]+)', expand = False)
 column = 'mrms'
gc[column] = gc['Name'].str.extract('([A-Za-z]+)\.', expand = False)
 gc[column] = gc[column].replace(['Col', 'Mlle', 'Major','Countess','Lady', 'Sir', 'Dona', 'Capt', 'Jonkheer', 'Don', 'Mme', 'Ms', 'Dr', 'Rev'], 'Other')
 le = LabelEncoder()
gc['Ticket_head']=le.fit_transform(gc['Ticket_head'])
 gc['Ticket_num']=le.fit_transform(gc['Ticket_num'] )<categorify>
model.fit(features, labels )
list_ce = ['Cabin','Embarked','Sex','mrms','Survived','Fare']
ce_ohe = ce.OneHotEncoder(cols=list_ce,handle_unknown='impute')
 gc = ce_ohe.fit_transform(gc )<drop_column>
N = 5
for Category, category_id in sorted(category_to_id.items()):
indices = np.argsort(model.coef_[category_id])
 feature_names = np.array(tfidf.get_feature_names())[indices]
unigrams = [v for v in reversed(feature_names)if len(v.split(' ')) == 1][:N]
bigrams = [v for v in reversed(feature_names)if len(v.split(' ')) == 2][:N]
print("
print(".Top unigrams: 
.{}".format(' 
.'.join(unigrams)))
 print(".Top bigrams: 
.{}".format(' 
.'.join(bigrams)) )
gc = gc.drop(['Name','Ticket'], axis=1 )<drop_column>
texts = ["Hooli stock price soared after a dip in PiedPiper revenue growth.",
"Captain Tsubasa scores a magnificent goal for the Japanese team.",
"Merryweather mercenaries are sent on another mission, as government oversight groups call for new sanctions.",
"Beyoncé releases a new album, tops the charts in all of south-east Asia!",
"You won't guess what the latest trend in data analysis is!"]
text_features = tfidf.transform(texts)
 predictions = model.predict(text_features)
 for text, predicted in zip(texts, predictions):
print('"{}"'.format(text))
print(" - Predicted as: '{}'".format(id_to_category[predicted]))
print("" )
gccollist.remove('Age' )<count_missing_values>
TEST_PATH = os.path.join(".. /input/bbc-test", "BBC News Test.csv")
 test_df = pd.read_csv(TEST_PATH)
gc.isnull().sum()<categorify>
test_df.Text.tolist()
gc['Age']=gc['Age'].fillna(gc.groupby(gccollist)['Age'].transform('mean'))<count_missing_values>
test_features = tfidf.transform(test_df.Text.tolist())
 Y_pred = model.predict(test_features )
gc.Age.isnull().sum()<filter>
Y_pred_name =[]
for cat_id in Y_pred :
Y_pred_name.append(id_to_category[cat_id] )
gc[gc['Age'].isnull() ]<create_dataframe>
submission = pd.DataFrame({
"ArticleId": test_df["ArticleId"],
"Category": Y_pred_name
} )
gc_corr = gc.corr()
corr_y = pd.DataFrame({"features":gc.columns,"corr_y":gc_corr["Age"]},index=None)
 corr_y = corr_y.reset_index(drop=True )<sort_values>
submission.to_csv('submission.csv', index=False )