Kaggle competition: Sarcasmo
<SOS> metric: auc Kaggle data source: sarcasmo<create_dataframe>
import numpy as np
import pandas as pd
import seaborn as sb
import matplotlib.pyplot as plt
import sklearn as sk
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score, roc_auc_score
from sklearn.model_selection import cross_validate
com = pd.DataFrame({'pre1':predictions1, 'pre2': predictions2} )<categorify>
o_train = pd.read_csv(".. /input/train.csv")
 o_valid = pd.read_csv(".. /input/valid.csv")
 train = pd.read_csv(".. /input/train.csv")
 valid = pd.read_csv(".. /input/valid.csv")
 data = pd.concat([train, valid], sort=False)
 example_sub = pd.read_csv(".. /input/sample_submission.csv" )
%matplotlib inline
def transform_dataset(ds):
transformed_dataset = ds.copy()
transformed_dataset['Age'].fillna(transformed_dataset['Age'].median() , inplace=True)
 transformed_dataset['Fare'].fillna(transformed_dataset['Fare'].median() , inplace=True)
 transformed_dataset['Sex'] = pd.factorize(transformed_dataset['Sex'])[0]
transformed_dataset=pd.concat([transformed_dataset, pd.get_dummies(transformed_dataset['Embarked'], prefix="Embarked")], axis=1)
 transformed_dataset['Childs'] = transformed_dataset.apply(lambda row: get_childs(row), axis=1)
 transformed_dataset['Parents'] = transformed_dataset.apply(lambda row: get_parents(row), axis=1)
 transformed_dataset = transformed_dataset.drop(['PassengerId', 'Name', 'Ticket', 'Cabin', 'Embarked', 'Parch'], axis=1)
 return transformed_dataset
def get_childs(row):
if(row.Parch>2 or row.Age>=18):
return row.Parch
return 0
def get_parents(row):
if(row.Parch <= 2 and row.Age<18):
return row.Parch
return 0<load_from_csv>
train[train.ID == 5022].article_link
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 passenger_id = test['PassengerId']<categorify>
sarcastic = len(train[train.is_sarcastic == 1])
 non_sarcastic = len(train[train.is_sarcastic == 0])
 sarcastic /(non_sarcastic + sarcastic )
transformed_train = transform_dataset(train)
 transformed_test = transform_dataset(test )<prepare_x_and_y>
print(np.where(pd.isnull(train)))
 print(np.where(pd.isna(train)))
 np.where(train.applymap(lambda x: x == ''))
X = transformed_train.drop(['Survived'], axis=1)
 Y = transformed_train['Survived']<split>
sarcasm_classfication = Pipeline([ 
('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('classify', LinearSVC(C=1))
] )
X_train, X_test, Y_train, Y_test = model_selection.train_test_split(X, Y, test_size = 0.2, random_state = 21 )<choose_model_class>
X_train, X_test, y_train, y_test = train_test_split(train.headline, train.is_sarcastic )
param_grid =[ {'n_estimators' : [4,5,10, 15, 20, 25, 30, 35, 40], 'max_depth' : [5,10,15, 20]},]
rf = ensemble.GradientBoostingClassifier(random_state=21)
 model = GridSearchCV(rf,param_grid, cv = 5 )<train_model>
sarcasm_classfication.fit(X_train, y_train )
model.fit(X_train,Y_train)
 print('train score = ', model.score(X_train,Y_train), '
test score = ', model.score(X_test,Y_test), '
', model.best_params_ )<predict_on_test>
print(roc_auc_score(y_train, sarcasm_classfication.decision_function(X_train)))
 print(roc_auc_score(y_test, sarcasm_classfication.decision_function(X_test)))
 cross_validate(sarcasm_classfication, train.headline, train.is_sarcastic, cv=5, scoring='roc_auc' )
Y_predict = model.predict(transformed_test)
 Y_p = pd.DataFrame(Y_predict, columns=['Survived'])
 Y_p<concatenate>
def remove_punctuation(dataframe):
rgx = '('s|[!?,.:;'$])'
tmp = dataframe.copy()
tmp['headline'] = tmp['headline'].str.replace(rgx, '')
 return tmp
res = pd.concat([passenger_id, Y_p], axis=1)
 res<save_to_csv>
tmp = train.copy()
tmp = remove_punctuation(tmp)
 X_train, X_test, y_train, y_test = train_test_split(tmp.headline, tmp.is_sarcastic)
 sarcasm_classfication.fit(X_train, y_train)
 print(roc_auc_score(y_train, sarcasm_classfication.decision_function(X_train)))
 print(roc_auc_score(y_test, sarcasm_classfication.decision_function(X_test)))
 cross_validate(sarcasm_classfication, X_train, y_train, cv=5, scoring='roc_auc' )
res.to_csv('res.csv', index=None )<categorify>
def get_top_n_words(corpus, n=None):
vec = CountVectorizer().fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx], idx)for word, idx in vec.vocabulary_.items() ]
words_freq = sorted(words_freq, key = lambda x: x[1], reverse=True)
 words = [x[0] for x in words_freq]
count = [x[1] for x in words_freq]
return pd.DataFrame({'Words': words[:n], 'Amount': count[:n]} )
%matplotlib inline
def transform_dataset(ds):
transformed_dataset = ds.copy()
transformed_dataset['Age'].fillna(transformed_dataset['Age'].median() , inplace=True)
 transformed_dataset['Fare'].fillna(transformed_dataset['Fare'].median() , inplace=True)
 transformed_dataset['Sex'] = pd.factorize(transformed_dataset['Sex'])[0]
transformed_dataset=pd.concat([transformed_dataset, pd.get_dummies(transformed_dataset['Embarked'], prefix="Embarked")], axis=1)
 transformed_dataset['Childs'] = transformed_dataset.apply(lambda row: get_childs(row), axis=1)
 transformed_dataset['Parents'] = transformed_dataset.apply(lambda row: get_parents(row), axis=1)
 transformed_dataset = transformed_dataset.drop(['PassengerId', 'Name', 'Ticket', 'Cabin', 'Embarked', 'Parch'], axis=1)
 return transformed_dataset
def get_childs(row):
if(row.Parch>2 or row.Age>=18):
return row.Parch
return 0
def get_parents(row):
if(row.Parch <= 2 and row.Age<18):
return row.Parch
return 0<load_from_csv>
t = train.copy()
for word in tmp.Words:
t.headline.str.replace(word, '')
 X_train, X_test, y_train, y_test = train_test_split(t.headline, t.is_sarcastic)
 sarcasm_classfication.fit(X_train, y_train)
 print(roc_auc_score(y_train, sarcasm_classfication.decision_function(X_train)))
 print(roc_auc_score(y_test, sarcasm_classfication.decision_function(X_test)))
 cross_validate(sarcasm_classfication, X_train, y_train, cv=5, scoring='roc_auc' )
dataset = pd.read_csv('.. /input/titanic/train.csv')
 dataset.info()<categorify>
def remove_double_links(series):
rgx = '(https?(?!.+https? ).+)'
tmp = series.copy()
tmp['article_link'] = tmp['article_link'].str.extract(rgx)
 return tmp
transformed_dataset = transform_dataset(dataset)
 transformed_dataset.info()<prepare_x_and_y>
def get_source(series):
rgx = '(( ?!https?:)\/\/.+?\.. +?\/)'
tmp = series.copy()
tmp['article_link'] = tmp['article_link'].str.extract(rgx)
 tmp['article_link'] = tmp['article_link'].str.strip(to_strip="/w")
 tmp['article_link'] = tmp['article_link'].str.strip(to_strip=r'^\.')
 return tmp
X = transformed_dataset.drop(['Survived'], axis=1)
 Y = dataset['Survived']<split>
def rename_article_link(dataframe):
return dataframe.rename(columns={'article_link': 'source'} )
X_train, X_test, Y_train, Y_test = model_selection.train_test_split(X, Y, test_size = 0.2, random_state = 25 )<choose_model_class>
train['headline_and_source'] = train.source + ' ' + train.headline
model = tree.DecisionTreeClassifier(max_depth=6,random_state=25 )<train_model>
X_train, X_test, y_train, y_test = train_test_split(train['headline_and_source'], train.is_sarcastic)
 sarcasm_classfication.fit(X_train, y_train )
model.fit(X_train,Y_train)
 print('train score = ', model.score(X_train,Y_train), '
test score = ', model.score(X_test,Y_test))<load_from_csv>
print(roc_auc_score(y_train, sarcasm_classfication.decision_function(X_train)))
 print(roc_auc_score(y_test, sarcasm_classfication.decision_function(X_test)))
 cross_validate(sarcasm_classfication, train.headline_and_source, train.is_sarcastic, cv=25, scoring='roc_auc', return_train_score=True )
test = pd.read_csv('.. /input/titanic/test.csv')
 test.info()<define_variables>
tmp = train.copy()
X_tmp_train, X_tmp_test, y_tmp_train, y_tmp_test = train_test_split(tmp.source, tmp.is_sarcastic)
 tmp_model = sarcasm_classfication
tmp_model.fit(X_tmp_train, y_tmp_train)
 print(roc_auc_score(y_tmp_train, tmp_model.decision_function(X_tmp_train)))
 print(roc_auc_score(y_tmp_test, tmp_model.decision_function(X_tmp_test)))
 cross_validate(sarcasm_classfication, X_tmp_train, y_tmp_train, cv=5, scoring='roc_auc', return_train_score=True )
passenger_id = test['PassengerId']<categorify>
tmp = train.copy()
tmp = tmp.drop(columns=['ID', 'headline', 'headline_and_source'])
 tmp = pd.get_dummies(data=tmp, columns=['source'])
 X_tmp_train, X_tmp_test, y_tmp_train, y_tmp_test = train_test_split(tmp.drop(columns=['is_sarcastic']), tmp.is_sarcastic)
 tmp_model = RandomForestClassifier(n_jobs=-1, n_estimators=100)
 tmp_model.fit(X_tmp_train, y_tmp_train)
 prob = tmp_model.predict_proba(X_tmp_train)
 prob = prob[:, 1]
print(roc_auc_score(y_tmp_train, prob))
prob = tmp_model.predict_proba(X_tmp_test)
 prob = prob[:, 1]
print(roc_auc_score(y_tmp_test, prob))
cross_validate(tmp_model, X_tmp_train, y_tmp_train, cv=5, scoring='roc_auc', return_train_score=True )
transformed_test = transform_dataset(test)
 transformed_test.info()<predict_on_test>
valid = remove_double_links(valid)
 valid = get_source(valid)
 valid = valid.rename(columns={'article_link': 'source'} )
<concatenate><EOS> 
predicted = sarcasm_classfication.predict(valid.source)
 prediction_dataframe = pd.DataFrame({'ID': valid.ID, 'is_sarcastic': predicted})
 prediction_dataframe.to_csv('output.csv', index=False )