Kaggle - Don't call me turkey!  (LSTM)

##Reference
[Don't call me turkey!](https://www.kaggle.com/c/dont-call-me-turkey)

##Library
```python
import pandas as pd
import numpy as np
import os

from sklearn.model_selection import train_test_split

from keras.preprocessing.sequence import pad_sequences

from keras.models import Sequential, Model
from keras.layers import Input, Dense, LSTM, Bidirectional, Dropout, BatchNormalization
from keras.layers import GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate

from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints
```

##Data
```python
train = pd.read_json('../input/train.json')

print ('shape train: ', train.shape)
train.head()
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/5b07cd6a-08b9-4dc9-d364-21f02be00dba.png)


```python
from IPython.display import YouTubeVideo

index = 1
YouTubeVideo(train['vid_id'][index],start=train['start_time_seconds_youtube_clip'][index],
             end=train['end_time_seconds_youtube_clip'][index])
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/18ad4ec9-ae5e-d5a3-59bf-d6630ee7a125.png)


```python
x_train = pad_sequences(train['audio_embedding'], maxlen=10, padding='post')
y_train = train['is_turkey']

print ('shape x_train: ', x_train.shape)
print ('shape y_train: ', y_train.shape)
print ('# of 1:', np.sum(y_train==1))
```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/7cdbbf54-fb90-726a-11ab-1dc6e2cbbe01.png)

```python
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.2, stratify=y_train)

print ('shape x_train: ', x_train.shape)
print ('shape x_val: ', x_val.shape)
```

##Model 1
```python
model = Sequential()
model.add(BatchNormalization(input_shape=(10, 128)))
model.add(Bidirectional(LSTM(64, return_sequences=True)))
model.add(Bidirectional(LSTM(64, return_sequences=False)))
model.add(Dense(1,activation='sigmoid'))

print (model.summary())
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/b8620f67-6a80-af4f-bf02-e1002f2018f2.png)


##Model 2
```python
model = Sequential()
model.add(BatchNormalization(input_shape=(10, 128)))
model.add(Bidirectional(LSTM(64, return_sequences=True)))
model.add(GlobalAveragePooling1D())
model.add(Dense(1,activation='sigmoid'))

print (model.summary())
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/d20425e3-b542-6b12-398a-45802997f218.png)

##Model 3
```python
inputs = Input(shape=(10,128))
x = BatchNormalization()(inputs)
x = Bidirectional(LSTM(64, return_sequences=True))(x)
ave_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
x = concatenate([ave_pool, max_pool])
x = Dense(64, activation='relu')(x)
x = Dropout(0.25)(x)
outputs = Dense(1,activation='sigmoid')(x)

model = Model(inputs=inputs, outputs=outputs)

print (model.summary())
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/d31377da-24ae-992c-d63c-4997b96bcff8.png)


##Model 4 (Attention)
```python
class Attention(Layer):
    def __init__(self, step_dim,
                 W_regularizer=None, b_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, **kwargs):
        self.supports_masking = True
        self.init = initializers.get('glorot_uniform')

        self.W_regularizer = regularizers.get(W_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)

        self.W_constraint = constraints.get(W_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        self.step_dim = step_dim
        self.features_dim = 0
        super(Attention, self).__init__(**kwargs)

    def build(self, input_shape):
        assert len(input_shape) == 3

        self.W = self.add_weight((input_shape[-1],),
                                 initializer=self.init,
                                 name='{}_W'.format(self.name),
                                 regularizer=self.W_regularizer,
                                 constraint=self.W_constraint)
        self.features_dim = input_shape[-1]

        if self.bias:
            self.b = self.add_weight((input_shape[1],),
                                     initializer='zero',
                                     name='{}_b'.format(self.name),
                                     regularizer=self.b_regularizer,
                                     constraint=self.b_constraint)
        else:
            self.b = None

        self.built = True

    def compute_mask(self, input, input_mask=None):
        return None

    def call(self, x, mask=None):
        features_dim = self.features_dim
        step_dim = self.step_dim

        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),
                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))

        if self.bias:
            eij += self.b

        eij = K.tanh(eij)

        a = K.exp(eij)

        if mask is not None:
            a *= K.cast(mask, K.floatx())

        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
        return K.sum(weighted_input, axis=1)

    def compute_output_shape(self, input_shape):
        return input_shape[0],  self.features_dim

model = Sequential()
model.add(BatchNormalization(input_shape=(10, 128)))
model.add(Bidirectional(LSTM(64, return_sequences=True)))
model.add(Attention(10))
model.add(Dense(1,activation='sigmoid'))

print (model.summary())
```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/d8195c54-8f55-c7f8-0b55-a5bebb13e06c.png)