Kaggle competition: Titanic - Machine Learning from Disaster
model.fit(features, labels )<features_selection>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()
N = 5
for Category, category_id in sorted(category_to_id.items()):
indices = np.argsort(model.coef_[category_id])
 feature_names = np.array(tfidf.get_feature_names())[indices]
unigrams = [v for v in reversed(feature_names)if len(v.split(' ')) == 1][:N]
bigrams = [v for v in reversed(feature_names)if len(v.split(' ')) == 2][:N]
print("
print(".Top unigrams: 
.{}".format(' 
.'.join(unigrams)))
 print(".Top bigrams: 
.{}".format(' 
.'.join(bigrams)) )<predict_on_test>
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()
texts = ["Hooli stock price soared after a dip in PiedPiper revenue growth.",
"Captain Tsubasa scores a magnificent goal for the Japanese team.",
"Merryweather mercenaries are sent on another mission, as government oversight groups call for new sanctions.",
"Beyoncé releases a new album, tops the charts in all of south-east Asia!",
"You won't guess what the latest trend in data analysis is!"]
text_features = tfidf.transform(texts)
 predictions = model.predict(text_features)
 for text, predicted in zip(texts, predictions):
print('"{}"'.format(text))
print(" - Predicted as: '{}'".format(id_to_category[predicted]))
print("" )<load_from_csv>
women = train_data.loc[train_data.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("% of women who survived:", rate_women )
TEST_PATH = os.path.join(".. /input/bbc-test", "BBC News Test.csv")
 test_df = pd.read_csv(TEST_PATH)
<data_type_conversions>
men = train_data.loc[train_data.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("% of men who survived:", rate_men )
test_df.Text.tolist()<predict_on_test>
print(train_data.isna().sum())
 print(test_data.isna().sum() )
test_features = tfidf.transform(test_df.Text.tolist())
 Y_pred = model.predict(test_features)
 Y_pred<define_variables>
import seaborn as sns
import matplotlib.pyplot as plt
Y_pred_name =[]
for cat_id in Y_pred :
Y_pred_name.append(id_to_category[cat_id] )<create_dataframe>
train_data['Survived'].value_counts()
submission = pd.DataFrame({
"ArticleId": test_df["ArticleId"],
"Category": Y_pred_name
} )<save_to_csv>
train_data['SibSp'].value_counts()
submission.to_csv('submission.csv', index=False )<load_from_csv>
train_data['Parch'].value_counts()
TRAIN_PATH = os.path.join(".. /input/ai-academy-intermediate-class-competition-1", "BBC News Train.csv")
 df = pd.read_csv(TRAIN_PATH )<feature_engineering>
print(train_data['Name'].value_counts())
 print(train_data['Ticket'].value_counts())
 print(train_data['Fare'].value_counts() )
df['category_id'] = df['Category'].factorize() [0]
df['category_id'][0:10]<remove_duplicates>
print(train_data['Age'].isna().sum())
 print(test_data['Age'].isna().sum() )
category_id_df = df[['Category', 'category_id']].drop_duplicates().sort_values('category_id' )<define_variables>
train_data['Age'].fillna(value=train_data['Age'].median() , inplace=True)
 print(train_data['Age'].isna().sum())
 test_data['Age'].fillna(value=test_data['Age'].median() , inplace=True)
 print(test_data['Age'].isna().sum() )
category_to_id = dict(category_id_df.values)
 id_to_category = dict(category_id_df[['category_id', 'Category']].values )<count_values>
train_data = train_data[train_data['Fare'] < 100]
df.groupby('Category' ).category_id.count() 
<categorify>
test_data['Fare'].isna().sum()
tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', encoding='latin-1', ngram_range=(1, 2), stop_words='english')
 features = tfidf.fit_transform(df.Text ).toarray()
labels = df.category_id 
<sort_values>
train_data['Embarked'].isna().sum()
sorted(category_to_id.items() )<statistical_test>
train_data['Embarked'].fillna(method = 'ffill' , inplace = True)
N = 3
for Category, category_id in sorted(category_to_id.items()):
features_chi2 = chi2(features, labels == category_id)
 indices = np.argsort(features_chi2[0])
 feature_names = np.array(tfidf.get_feature_names())[indices]
unigrams = [v for v in feature_names if len(v.split(' ')) == 1]
bigrams = [v for v in feature_names if len(v.split(' ')) == 2]
print("
print(".Most correlated unigrams: 
.{}".format(' 
.'.join(unigrams[-N:])))
 print(".Most correlated bigrams: 
.{}".format(' 
.'.join(bigrams[-N:])) )<define_variables>
y = train_data['Survived']
features = ['Pclass', 'Sex', 'SibSp', 'Parch']
X = pd.get_dummies(train_data[features])
 X_test = pd.get_dummies(test_data[features])
 X = pd.concat([train_data[['Age', 'Fare']], X], axis=1)
 X_test = pd.concat([test_data[['Age','Fare']], X_test], axis=1)
 print(X)
 print(X_test)
SAMPLE_SIZE = int(len(features)* 0.3)
 np.random.seed(0)
 indices = np.random.choice(range(len(features)) , size=SAMPLE_SIZE, replace=False)
 projected_features = TSNE(n_components=2, random_state=0 ).fit_transform(features[indices])
<filter>
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
my_id = 0
projected_features[(labels[indices] == my_id ).values]<choose_model_class>
models = [
RandomForestClassifier(n_estimators=200, max_depth=3, random_state=0),
MultinomialNB() ,
LogisticRegression(random_state=0),
] 
<create_dataframe>
model_RFC = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=1)
 model_RFC.fit(X, y)
 predictions = model_RFC.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )
<find_best_model_class><EOS>