Kaggle competition: Sarcasmo
<SOS> metric: auc Kaggle data source: sarcasmo<save_to_csv>
%matplotlib inline
print(os.listdir(".. /input"))
res.to_csv('res.csv', index=None )<load_pretrained>
file_train = '.. /input/train.csv'
file_valid = '.. /input/valid.csv'
file_exemplo = '.. /input/sample_su.csv'
data_train = pd.read_csv(file_train)
 data_valid = pd.read_csv(file_valid)
 print(len(data_train), len(data_valid))
eli5.explain_weights_sklearn(model, feature_names=X_train.columns.values )<count_unique_values>
Y_dummies = pd.get_dummies(data_train['is_sarcastic'] ).values
Y = data_train['is_sarcastic']
data_train_format = data_train.drop('is_sarcastic', axis=1)
 data_all = data_train_format.append(data_valid)
 data_all = data_all.drop('article_link', axis=1)
 data_all = data_all.drop('ID', axis=1)
 data_all['headline'] = data_all['headline'].apply(lambda x: x.lower())
 data_all['headline'] = data_all['headline'].apply(( lambda x: re.sub('[^a-zA-z0-9\s]','',x)))
 print(len(data_train['headline']), len(data_valid['headline']))
print(len(data_all))
raw_data = pd.read_csv('.. /input/train.csv')
 print(raw_data.columns)
 raw_data.head()
print(raw_data.groupby(['Pclass', 'Sex', 'Embarked'])['Survived'].mean())
 print(raw_data.Cabin.nunique())
 print(raw_data.isnull().sum())
<categorify>
max_fatures = 5000
tokenizer = Tokenizer(num_words=max_fatures, split=' ')
 tokenizer.fit_on_texts(data_all['headline'].values)
 X = tokenizer.texts_to_sequences(data_all['headline'].values)
 X = pad_sequences(X )
def cabinMap(elem):
d = {'A': 0, 'B': 1, 'C': 2, 'D': 3, 'E': 4, 'F': 5, 'G': 6, 'T': 7, 'N': 8}
elem = d[elem[0]]
return elem 
<prepare_x_and_y>
data_train_split = X[:len(data_train)]
data_valid_split = X[len(data_train):]
print(len(data_train_split), len(data_valid_split))
y = raw_data.Survived
features = ['Pclass', 'Sex', 'Embarked']
X = raw_data[features].copy()
X.groupby('Sex' ).count()<categorify>
embed_dim = 64
lstm_out = 196
model = Sequential()
model.add(Embedding(max_fatures, embed_dim,input_length = data_train_split.shape[1]))
model.add(LSTM(lstm_out, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(2,activation='softmax'))
model.compile(loss = 'categorical_crossentropy', optimizer='adam', metrics = ['accuracy'])
def reformat(x):
d_sex = {'female': 0, 'male': 1}
d_emb = {'C': 0, 'Q': 1, 'S': 2, '3':3}
x['Sex'] = [d_sex[sex] for sex in x['Sex'].values]
x['Embarked'] = x['Embarked'].fillna('3')
 x['Embarked'] = [d_emb[elem] for elem in x['Embarked'].values]
try:
x['Cabin'] = x['Cabin'].fillna('N')
 x['Cabin'] = x['Cabin'].apply(cabinMap)
 except KeyError:
pass
reformat(X)
 X.head()<split>
batch_size = 32
history = model.fit(data_train_split, Y_dummies, epochs = 60, batch_size=batch_size, verbose = 2 )
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1 )<normalization>
Y_sarcasm = model.predict(data_valid_split,batch_size=1,verbose = 2 )
<predict_on_test><EOS> 
data_to_submit = pd.DataFrame({
'ID': data_valid['ID'],
'is_sarcastic':list_sarcasm
})
 data_to_submit.to_csv('csv_to_submit.csv', index = False )