Kaggle competition: Titanic - Machine Learning from Disaster
finalModel = xgb.XGBClassifier(max_depth=10, learning_rate=0.1, n_estimators=1000, n_jobs=50)
 finalModel<train_model>
train = pd.read_csv('/kaggle/input/train.csv')
 train.head()
finalModel.fit(X_train, y_train, eval_metric='mae' )<predict_on_test>
temp = train
y_pred = finalModel.predict(X_test)
 print("Mean squared error: %.2f" % mean_squared_error(y_test, y_pred))
print('Variance score: %.2f' % r2_score(y_test, y_pred))<predict_on_test>
train.drop('Cabin',inplace=True, axis=1 )
envio_final = pd.DataFrame(teste_base["Property Id"])
 envio_final['score'] = finalModel.predict(teste ).round()
envio_final['score'] = envio_final["score"].astype(int)
 sb.countplot(x='score',data=envio_final)
 envio_final.describe().T<save_to_csv>
train.drop('PassengerId',inplace=True, axis=1)
 train.drop('Ticket',inplace=True, axis=1 )
envio_final.to_csv('final.csv', index=False )<install_modules>
def fill_age(columns):
Age = columns[0]
Pclass = columns[1]
if pd.isnull(Age):
if Pclass == 1 :
return 37.0
elif Pclass == 2 :
return 29.0
else :
return 24.0
else:
return Age
!pip install pymystem3<define_variables>
train['Age'] = train[['Age','Pclass']].apply(fill_age,axis=1 )
RAND_STATE = 37
N_JOBS = -1
VERB_LEVEL = 2
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
names = train.Name.str.split(',')
 names2 = []
for i in range(0,891):
names2.append(names[i][1].split('.')[0] )
df_train = pd.read_csv('/kaggle/input/ocrv-intent-classification/train.csv', index_col='id')
 df_train.head()<filter>
namedummies = pd.get_dummies(names2,drop_first=True )
df_train[df_train.text.isna() ]<remove_duplicates>
train = pd.concat([train,namedummies],axis=1 )
df_train = df_train.drop_duplicates()
df_train.info()<load_from_csv>
sex = pd.get_dummies(train['Sex'],drop_first=True)
 emb = pd.get_dummies(train['Embarked'],drop_first=True )
df_test = pd.read_csv('/kaggle/input/ocrv-intent-classification/test.csv', index_col='id' )<string_transform>
train.drop(['Sex','Embarked','Name'],inplace=True, axis=1 )
df_test = df_test.fillna(' ' )<categorify>
train = pd.concat([train,sex,emb],axis=1);
lemmatizator = Mystem()
mystem_preprocessor = lambda x: ''.join(lemmatizator.lemmatize(x)[:-1])
 X_train = df_train.text.apply(mystem_preprocessor )<prepare_x_and_y>
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
y_train = df_train.label<feature_engineering>
X_train, X_test, y_train, y_test = train_test_split(train.drop(['Fare','Survived'],axis=1), train['Survived'], test_size=0.33, random_state=42 )
X_test = df_test.text.apply(mystem_preprocessor )<load_pretrained>
logmodel = LogisticRegression()
logmodel.fit(X_train,y_train);
nltk.download('stopwords')
 russian_stopwords = stopwords.words('russian' )<train_on_grid>
pred = logmodel.predict(X_test )
mnnb_clf = Pipeline(
 [('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('mnnb', MultinomialNB()),])
 parameters = {
'vect__ngram_range': [(1,1),(1,2),(1,3),],
'vect__min_df': [1,],
'vect__max_df': [1.],
'vect__stop_words': [None, russian_stopwords],
'tfidf__use_idf': [True, False],
'mnnb__alpha': [.1,.01,.001]}
gs_mnnb_clf= GridSearchCV(mnnb_clf, parameters, n_jobs=N_JOBS, verbose=VERB_LEVEL)
 gs_mnnb_clf = gs_mnnb_clf.fit(X_train, y_train)
 print(gs_mnnb_clf.best_score_, gs_mnnb_clf.best_params_ )<save_to_csv>
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
def predict_submit(gs, df, name_suf):
y_pred = gs.predict(df)
 submission = pd.DataFrame(y_pred, columns=['label'])
 submission.index.name = 'id'
submission.to_csv(f'submission_{name_suf}.csv' )<predict_on_test>
test = pd.read_csv('.. /input/test.csv')
 temp_test = test
predict_submit(gs_mnnb_clf, X_test, name_suf='mnnb' )<define_search_model>
pass_id = test.PassengerId
test.drop(['Cabin','PassengerId','Ticket'],inplace=True, axis=1 )
sgd_clf = Pipeline(
 [('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('sgdc', SGDClassifier(random_state=RAND_STATE))
])
 parameters = {
'vect__ngram_range': [(1,3),],
'vect__stop_words': [None, russian_stopwords],
'tfidf__use_idf': [True, False],
'sgdc__loss': ['hinge', 'perceptron'],
'sgdc__penalty': ['l1', 'l2', 'elasticnet'],
'sgdc__alpha': [1.25e-5, 2.5e-5, 5e-5]}
gs_sgd_clf = GridSearchCV(sgd_clf, parameters, n_jobs=N_JOBS, verbose=VERB_LEVEL)
 gs_sgd_clf = gs_sgd_clf.fit(X_train, y_train)
 print(gs_sgd_clf.best_score_, gs_sgd_clf.best_params_ )<predict_on_test>
print("The mean before is ", test.Age.mean())
 print(test.groupby('Pclass' ).mean() ['Age'] )
predict_submit(gs_sgd_clf, X_test, name_suf='sgd' )<define_search_model>
def age_fill(columns):
Age = columns[0]
Pclass = columns[1]
if pd.isnull(Age):
if Pclass == 1 :
return 40.69
elif Pclass == 2 :
return 28.83
else :
return 24.39
else:
return Age
logreg_clf = Pipeline(
 [('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('logreg', LogisticRegression(random_state=RAND_STATE)) ,])
 parameters = {
'vect__ngram_range': [(1,3),],
'vect__min_df': [1,],
'vect__max_df': [1.],
'vect__stop_words': [None, russian_stopwords],
'tfidf__use_idf': [True, False],
'logreg__C': [11,17,23],
'logreg__multi_class': ['ovr', 'multinomial'],
'logreg__solver': ['lbfgs', 'newton-cg']}
gs_logreg_clf = GridSearchCV(logreg_clf, parameters, n_jobs=N_JOBS, verbose=VERB_LEVEL)
 gs_logreg_clf = gs_logreg_clf.fit(X_train, y_train)
 print(gs_logreg_clf.best_score_, gs_logreg_clf.best_params_ )<predict_on_test>
test['Age'] = test[['Age','Pclass']].apply(age_fill,axis=1 )
predict_submit(gs_logreg_clf, X_test, name_suf='logreg' )<feature_engineering>
namesx = test.Name.str.split(',')
 namesx2 = []
for i in range(0,test.shape[0]):
namesx2.append(namesx[i][1].split('.')[0] )
word2vec_path = '.. /input/web-mystem-skipgram-500-2-2015/web.bin'
word2vec_size = 500
word2vec = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True)
 words = word2vec.index2word
w_rank = {}
for i,word in enumerate(words):
word = word.split('_')[0]
w_rank[word] = i
WORDS = w_rank
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or known(edits2(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`."
return(e2 for e1 in edits1(word)for e2 in edits1(e1))<install_modules>
namexdummies = pd.get_dummies(namesx2,drop_first=True )
!pip install pymorphy2<string_transform>
test = pd.concat([test,namexdummies],axis=1 )
def spell_norm(ser):
tokenizer = RegexpTokenizer(r'[а-яА-Я]+')
 morph = pymorphy2.MorphAnalyzer()
normolize = lambda t: morph.parse(t)[0].normal_form
def spelling_correct(token):
methods_stack = morph.parse(token)[0].methods_stack
if str(morph.parse(token)[0].methods_stack[0][0])!= '<DictionaryAnalyzer>':
return False
for method in methods_stack[1:]:
if 'unknown' in str(method[0] ).lower() :
return False
return True
def norm_spelling(tokens):
lemmas = []
for token in tokens:
if not spelling_correct(token):
token = correction(token)
 lemma = normolize(token)
 lemmas.append(lemma)
 return lemmas
return ser.apply(tokenizer.tokenize ).apply(norm_spelling)
 spell_norm(df_train.text[10:20] )<feature_engineering>
sex1 = pd.get_dummies(test['Sex'],drop_first=True)
 emb1 = pd.get_dummies(test['Embarked'],drop_first=True)
 test.drop(['Sex','Embarked','Name'],inplace=True, axis=1)
 test = pd.concat([test,sex1,emb1],axis=1);
%%time
df_train['spell_norm_text'] = spell_norm(df_train.text ).apply(lambda x: ' '.join(x))<prepare_x_and_y>
test.fillna(test['Fare'].median() ,inplace=True )
X_train, y_train = df_train.spell_norm_text, df_train.label<feature_engineering>
new_train = train.drop([' Jonkheer',' the Countess',' Mme',' Mlle',' Major',' Lady',' Col', ' Don', ' Sir'],axis=1)
 new_test = test.drop([' Dona'],axis=1 )
%%time
df_test['spell_norm_text'] = spell_norm(df_test.text ).apply(lambda x: ' '.join(x))<prepare_x_and_y>
X_train1, X_test1, y_train1, y_test1 = train_test_split(new_train.drop(['Survived'],axis=1), new_train['Survived'], test_size=0.33, random_state=42 )
X_test = df_test.spell_norm_text<train_on_grid>
new_logmodel = LogisticRegression()
new_logmodel.fit(X_train1,y_train1 )
sgd_spnorm_clf = Pipeline(
 [('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('sgdc', SGDClassifier(random_state=RAND_STATE))
])
 parameters = {
'vect__ngram_range': [(1,3),],
'vect__stop_words': [None,],
'tfidf__use_idf': [False, ],
'sgdc__loss': ['hinge', 'perceptron'],
'sgdc__penalty': ['l1', 'l2', 'elasticnet'],
'sgdc__alpha': [5e-6, 1.25e-5, 2.5e-5, 5e-5],
'sgdc__class_weight': [None, 'balanced']}
gs_sgd_spnorm_clf = GridSearchCV(sgd_spnorm_clf, parameters, n_jobs=N_JOBS, verbose=VERB_LEVEL)
 gs_sgd_spnorm_clf = gs_sgd_spnorm_clf.fit(X_train, y_train)
 print(gs_sgd_spnorm_clf.best_score_, gs_sgd_spnorm_clf.best_params_ )<predict_on_test>
prediction = new_logmodel.predict(new_test )
predict_submit(gs_sgd_spnorm_clf, X_test, name_suf='sgd_spnorm' )<define_search_space>
output = pd.DataFrame({ 'PassengerId' : pass_id, 'Survived': prediction } )
logreg_spnorm_clf = Pipeline(
 [('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('logreg', LogisticRegression(random_state=RAND_STATE)) ,])
 parameters = {
'vect__ngram_range': [(1,3),],
'vect__min_df': [1,],
'vect__max_df': [1.],
'vect__stop_words': [None,],
'tfidf__use_idf': [False,],
'logreg__C': [8,11,17],
'logreg__multi_class': ['ovr', 'multinomial'],
'logreg__solver': ['lbfgs', 'newton-cg'],
'logreg__class_weight': [None, 'balanced']}
gs_logreg_spnorm_clf = GridSearchCV(logreg_spnorm_clf, parameters, n_jobs=N_JOBS, verbose=VERB_LEVEL)
 gs_logreg_spnorm_clf = gs_logreg_spnorm_clf.fit(X_train, y_train)
 print(gs_logreg_spnorm_clf.best_score_, gs_logreg_spnorm_clf.best_params_ )<predict_on_test>
output.to_csv('titanic-predictions.csv', index = False )
predict_submit(gs_logreg_spnorm_clf, X_test, name_suf='logreg_spnorm' )<feature_engineering>
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
vectorizer = TfidfVectorizer(lowercase=True, analyzer='word', ngram_range=(1,3), use_idf=False)
 train_vectors = vectorizer.fit_transform(df_train.spell_norm_text.apply(lambda tr_vect: np.str_(tr_vect)) )<string_transform>
print(classification_report(y_test1,pred))
def tagged_tokens(ser, tokenizer=RegexpTokenizer(r'\w+'), lemmatizator=Mystem()):
def lemteg(tokens, tokenizer=tokenizer, lemmatizator=lemmatizator):
lemtegs = []
for token in tokens:
try:
tag = lemmatizator.analyze(token)[0]['analysis'][0]['gr'].split(',')[0].split('=')[0]
except:
tag = 'XXX'
lemtegs.append(f'{token}_{tag}')
 return lemtegs
return ser.apply(tokenizer.tokenize ).apply(lemteg)
 tagged_tokens(df_train.spell_norm_text[:10] )<feature_engineering>
score = logmodel.score(X_test, y_test)
 print("Accuracy for train.csv is ",score*100,"%" )
df_train['tagged_tokens']= tagged_tokens(df_train.spell_norm_text )<normalization>
score1 = new_logmodel.score(X_test1, y_test1)
 print("Accuracy for test.csv is ",score1*100,"%" )