Kaggle competition: Movie Genre Classification
<SOS> metric: categorizationaccuracy Kaggle data source: movie-genre-classification<set_options>
def reset_tf_session() :
curr_session = tf.get_default_session()
if curr_session is not None:
curr_session.close()
K.clear_session()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
s = tf.InteractiveSession(config=config)
 K.set_session(s)
 return s
%matplotlib inline
sns.set_context('notebook' )<load_from_disk>
train_data_np = pd.read_csv(".. /input/train_data.txt",delimiter=' ::: ',header=None,names=['id','title','genre','desc'])
 predict_data_np = pd.read_csv(".. /input/test_data.txt",delimiter=' ::: ',header=None,names=['id','title','desc'])
 train_data_np.shape
df_train_0 = pd.read_json('.. /input/train.json')
 df_test_0 = pd.read_json('.. /input/test.json')
 display(df_train_0.head() )<create_dataframe>
lb = LabelBinarizer()
lb.fit(genres)
 y = lb.transform(train_data_np.genre)
 print(lb.classes_)
 print(train_data_np.genre[0])
 y[0]
print(df_train_0.isnull().sum())
 print(df_test_0.isnull().sum())
 df_train = df_train_0.copy()
df_test = df_test_0.copy()<filter>
desc = train_data_np['desc'].values
tokenizer = Tokenizer(num_words=None,lower=True)
 tokenizer.fit_on_texts(desc)
 deleted = 0
high_count_words = [w for w,c in tokenizer.word_counts.items() if c > 10.0*train_data_np.shape[0]]
for w in high_count_words:
del tokenizer.word_index[w]
del tokenizer.word_docs[w]
del tokenizer.word_counts[w]
deleted+=1
print("Delete ", w)
 print("Delete ", deleted, " words from tokenizer")
 desc_train, desc_test, y_train, y_test = train_test_split(desc, y, test_size=0.05, random_state=1000)
 X_train = tokenizer.texts_to_sequences(desc_train)
 X_test = tokenizer.texts_to_sequences(desc_test)
 vocab_size = len(tokenizer.word_index)+ 1
print(desc_train[2])
 print(X_train[2])
display(df_train.loc[df_train.num_ing < 2] )<filter>
maxlen = 100
X_train = pad_sequences(X_train, padding='post', maxlen=maxlen)
 X_test = pad_sequences(X_test, padding='post', maxlen=maxlen)
 print(X_train[0, :] )
display(df_train.loc[df_train.num_ing > 60] )<filter>
def make_model_8() :
s = reset_tf_session()
embedding_dim = 50
model = Sequential()
model.add(layers.Embedding(input_dim=vocab_size,
output_dim=embedding_dim,
input_length=maxlen))
model.add(keras.layers.GlobalAveragePooling1D())
 model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(len(genres), activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
 model.summary()
return model
gtr_60_ind = df_train[df_train['num_ing'] > 60].index.values
print(df_train.ingredients[gtr_60_ind].tolist() )<filter>
def make_model_9() :
s = reset_tf_session()
embedding_dim = 50
model = Sequential()
model.add(layers.Embedding(input_dim=vocab_size,
output_dim=embedding_dim,
input_length=maxlen))
model.add(layers.Bidirectional(layers.LSTM(100, return_sequences=True, dropout=0.7,recurrent_dropout=0.7)))
 model.add(layers.Flatten())
 model.add(layers.Dense(len(genres), activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
 model.summary()
return model
display(df_train.loc[df_train.num_ing > 40])
 print(df_train.ingredients[26103] )<filter>
model = make_model_8()
history = model.fit(X_train, y_train,
epochs=20,
verbose=True,
validation_data=(X_test, y_test),
batch_size=1000)
 plot_history(history )
df_train = df_train[df_train['num_ing'] < 32]<drop_column>
y_predict_train = model.predict_proba(X_train )
df_train.reset_index(inplace=True, drop=True )<choose_model_class>
T=201
y_predict = y_predict_train
print(y_predict[T,:])
 y_predict_max = np.argmax(y_predict[T,:])
 print(y_predict_max, lb.classes_[y_predict_max], y_predict[T,y_predict_max])
 print(np.argmax(y_train[T]),y_train[T])
 print(train_data_np.genre[T])
lemmatizer = nltk.WordNetLemmatizer()
print(lemmatizer.lemmatize('tomatoes'))<string_transform>
desc_predict = predict_data_np['desc'].values
X_predict = tokenizer.texts_to_sequences(desc_predict)
 X_predict = pad_sequences(X_predict, padding='post', maxlen=maxlen)
 y_predict = model.predict_proba(X_predict)
def ingredients(df, i):
ing_list = df['ingredients'].iloc[i].copy()
ing_concat = []
for ing in ing_list:
ing = u"".join([c for c in unicodedata.normalize('NFKD', ing)if not unicodedata.combining(c)])
 ing = re.sub(r'[^a-zA-Z]',' ', ing)
 if 'fish sauce' in ing:
ing = 'fishsauce'
if 'corn starch' in ing:
ing = 'cornstarch'
if 'sour cream' in ing:
ing = 'sourcream'
word_list = ing.split()
for word in word_list:
word = word.strip()
lem = lemmatizer.lemmatize(word)
 ing_concat.append(lem)
 ing_concat = list(np.unique(ing_concat))
df.at[i, 'ingredients_concat'] = ' '.join(ing_concat)
<categorify>
T=100
print(predict_data_np.iloc[T])
 y_predict_max = np.argmax(y_predict[T,:])
 print(y_predict_max, lb.classes_[y_predict_max], y_predict[T,y_predict_max] )
for i in range(df_train.shape[0]):
ingredients(df_train, i)
 for i in range(df_test.shape[0]):
ingredients(df_test, i )<feature_engineering>
genres_predict = []
ids = []
for i in range(y_predict.shape[0]):
y_predict_max = np.argmax(y_predict[i,:])
 genres_predict.extend([lb.classes_[y_predict_max]])
 submission = pd.DataFrame({'id':predict_data_np['id'].values,
'genre':genres_predict,
'title':predict_data_np['title'].values},
columns=['id', 'genre','title'])
 submission.to_csv('submission.csv', index=False,columns=['id', 'genre'])
 print('Save submit')
 submission.head()