Kaggle competition: Titanic - Machine Learning from Disaster
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 test['target']=-1
df = pd.concat([train ,test] )<set_options>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()
del train, test; gc.collect() ; time.sleep(5 )<split>
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 if file == '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in tqdm(open(file)) if len(o)>100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in tqdm(open(file, encoding='latin')))
 return embeddings_index<load_pretrained>
women = train_data.loc[train_data.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("% de mujeres que sobrevivieron:", rate_women )
embed_glove = load_embed('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt' )<load_pretrained>
men = train_data.loc[train_data.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("% de hombres que sobrevivieron:", rate_men )
embed_paragram = load_embed('.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt' )<categorify>
print('Total hombres:',len(men))
print('Total mujeres:',len(women))
print('Total sobrevivientes hombres:',sum(men))
print('Total sobrevivientes mujeres:',sum(women))
my_embedding_matrix = dict()
for k1,v1 in embed_glove.items() :
my_val = v1
if k1 in embed_paragram.keys() :
my_val =(v1 + embed_paragram[k1])/2
my_embedding_matrix[k1] = my_val
for k1,v1 in embed_paragram.items() :
if k1 not in embed_glove.keys() :
my_embedding_matrix[k1] = v1<load_pretrained>
parch_0 = train_data.loc[train_data.Parch == 0]["Survived"]
rate_parch_0 = sum(parch_0)/len(parch_0)
 print("% parch = 0 que sobrevivieron:", rate_parch_0)
 parch_1 = train_data.loc[train_data.Parch == 1]["Survived"]
rate_parch_1 = sum(parch_1)/len(parch_1)
 print("% parch = 1 que sobrevivieron:", rate_parch_1)
 parch_2 = train_data.loc[train_data.Parch == 2]["Survived"]
rate_parch_2 = sum(parch_2)/len(parch_2)
 print("% parch = 2 que sobrevivieron:", rate_parch_2)
 parch_3 = train_data.loc[train_data.Parch == 3]["Survived"]
rate_parch_3 = sum(parch_3)/len(parch_3)
 print("% parch = 3 que sobrevivieron:", rate_parch_3)
 parch_4 = train_data.loc[train_data.Parch == 4]["Survived"]
rate_parch_4 = sum(parch_4)/len(parch_4)
 print("% parch = 4 que sobrevivieron:", rate_parch_4)
 parch_5 = train_data.loc[train_data.Parch == 5]["Survived"]
rate_parch_5 = sum(parch_5)/len(parch_5)
 print("% parch = 5 que sobrevivieron:", rate_parch_5)
 parch_6 = train_data.loc[train_data.Parch == 6]["Survived"]
rate_parch_6 = sum(parch_6)/len(parch_6)
 print("% parch = 3 que sobrevivieron:", rate_parch_6)
 print('----')
 print('Parch0 sobrevivientes sobre el total',sum(parch_0)/len(train_data['Parch']))
print('Parch1 sobrevivientes sobre el total',sum(parch_1)/len(train_data['Parch']))
print('Parch2 sobrevivientes sobre el total',sum(parch_2)/len(train_data['Parch']))
print('Parch3 sobrevivientes sobre el total',sum(parch_3)/len(train_data['Parch']))
print('Parch4 sobrevivientes sobre el total',sum(parch_4)/len(train_data['Parch']))
print('Parch5 sobrevivientes sobre el total',sum(parch_5)/len(train_data['Parch']))
print('Parch6 sobrevivientes sobre el total',sum(parch_6)/len(train_data['Parch']))
print(sum(train_data['Survived'])/len(train_data['Survived']))
<feature_engineering>
print('Vacios en embarked:',train_data['Embarked'].isnull().values.any())
 print('Vacios en Sex:',train_data['Sex'].isnull().values.any())
 print('Vacios en Name:',train_data['Name'].isnull().values.any())
 print('Vacios en Age:',train_data['Age'].isnull().values.any())
 print('Vacios en Cabin:',train_data['Cabin'].isnull().values.any())
 print('Vacios en Ticket:',train_data['Ticket'].isnull().values.any())
df['size'] = df['question_text'].str.len()
print(mean(df['size']))
print(median(df['size']))
print(stdev(df['size']))
print(amax(df['size']))
print(amin(df['size']))
df = df.drop(['size'], axis=1 )<feature_engineering>
print('Embarked:',train_data['Embarked'].isnull().sum())
 print('Age:',train_data['Age'].isnull().sum())
 print('Cabin:',train_data['Cabin'].isnull().sum() )
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab<define_variables>
embarked_s = train_data.loc[train_data.Embarked == 'S']["Survived"]
rate_embarked_s = sum(embarked_s)/len(train_data['Survived'])
 print('% que sobrevivio con embarked = S: ',rate_embarked_s)
 embarked_c = train_data.loc[train_data.Embarked == 'C']["Survived"]
rate_embarked_c = sum(embarked_c)/len(train_data['Survived'])
 print('% que sobrevivio con embarked = C:',rate_embarked_c)
 embarked_q = train_data.loc[train_data.Embarked == 'Q']["Survived"]
rate_embarked_q = sum(embarked_q)/len(train_data['Survived'])
 print('% que sobrevivio con embarked = Q:',rate_embarked_q)
 print('s:',len(embarked_s), 'c:',len(embarked_c), 'q:',len(embarked_q))
embarked_s = train_data.loc[train_data.Embarked == 'S']["Survived"]
rate_embarked_s = sum(embarked_s)/len(embarked_s)
 print('%S: ',rate_embarked_s)
 embarked_c = train_data.loc[train_data.Embarked == 'C']["Survived"]
rate_embarked_c = sum(embarked_c)/len(embarked_c)
 print('%C:',rate_embarked_c)
 embarked_q = train_data.loc[train_data.Embarked == 'Q']["Survived"]
rate_embarked_q = sum(embarked_q)/len(embarked_q)
 print('%Q:',rate_embarked_q )
vocab = build_vocab(df['question_text'] )<feature_engineering>
train_data[train_data['Embarked'].isnull() ]
df['question_text'] = df['question_text'].apply(lambda x: x.lower() )<feature_engineering>
train_data[train_data['Age'].isnull() ]
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1<feature_engineering>
primera_clase = train_data.loc[train_data.Pclass == 1]["Survived"]
rate_1 = sum(primera_clase)/len(primera_clase)
 print("% Sobrevivientes en primera clase:", rate_1)
 segunda_clase = train_data.loc[train_data.Pclass == 2]["Survived"]
rate_2 = sum(segunda_clase)/len(segunda_clase)
 print("% Sobrevivientes en segunda clase:", rate_2)
 tercera_clase = train_data.loc[train_data.Pclass == 3]["Survived"]
rate_3 = sum(tercera_clase)/len(tercera_clase)
 print("% Sobrevivientes en tercera clase:", rate_3)
for index, row in df.iterrows() :
for w in paraules_prohibides:
if w in row['question_text']:
row['conte_paraula_prohibida']=1
break;<train_model>
train_data['Embarked'].fillna('S', inplace=True)
 train_data[(train_data['PassengerId'] == 62)|(train_data['PassengerId'] == 830)]
if "sexo" in paraules_prohibides:
print("OK")
 if "casa" in paraules_prohibides:
print("OK2" )<define_variables>
train_data['Cabin'].fillna('M', inplace=True)
 train_data['Cabin_full'] = train_data.Cabin.str.slice(0, 1)
 test_data['Cabin'].fillna('M', inplace=True)
 test_data['Cabin_full'] = test_data.Cabin.str.slice(0, 1 )
for index, row in df.iterrows() :
if row['conte_paraula_prohibida']==1 & index < 20:
print(row['question_text'] )<define_variables>
train_data[train_data['Cabin'].isnull() ]
my_embedding_matrix = {k: v for k, v in my_embedding_matrix.items() if k in vocab}<set_options>
test_data['age_bins'] = pd.cut(x=test_data['Age'], bins=[0, 12, 20, 25, 40, 80])
 test_data['age_bins'].unique()
del embed_paragram, embed_glove, vocab; gc.collect() ; time.sleep(5 )<define_variables>
train_data.corr(method ='pearson' )
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }<string_transform>
train_data['vaSolo'] = train_data['SibSp'] + train_data['Parch']
train_data['vaSolo'] = np.where(train_data['vaSolo'] == 0, 1, 0)
 train_data.head()
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text<drop_column>
test_data['vaSolo'] = test_data['SibSp'] + test_data['Parch']
test_data['vaSolo'] = np.where(test_data['vaSolo'] == 0, 1, 0)
 train_data.head()
df['question_text'] = df['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))<define_variables>
features = ['Pclass', 'Sex', 'Embarked', 'age_bins', 'vaSolo', 'Cabin_full']
y = train_data['Survived'].values
X = pd.get_dummies(train_data[features])
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.40, random_state=5, stratify=y)
 X_test_r = pd.get_dummies(test_data[features])
 model = RandomForestClassifier(n_estimators=300, max_depth=4, random_state=0, max_features=4)
 model.fit(X_train, y_train)
 y_pred= model.predict(X_test)
 model.score(X_test, y_test )
punct = "/-'?!., 
<feature_engineering>
X_test_r.head()
X_test_r['Cabin_full_T'] = 0
df['size'] = df['question_text'].str.len()
print(mean(df['size']))
print(median(df['size']))
print(stdev(df['size']))
print(amax(df['size']))
print(amin(df['size']))<define_variables>
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
for feature in zip(features, model.feature_importances_):
print(feature)
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "eur","$": "usd", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', } 
<categorify>
predictions = model.predict(X_test_r )
<feature_engineering><EOS> 
output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Se guardó el csv")