Kaggle competition: Digit Recognizer
print('Number of languages for a movie:')
 train['spoken_languages'].apply(lambda x: len(x)if x!={} else 0 ).value_counts()<count_unique_values>
%matplotlib inline
list_of_langs = list(train['spoken_languages'].apply(lambda x: [i['name'] for i in x]
if x!= {} else []))
top_langs = [m[0] for m in Counter(i for j in list_of_langs for i in j ).most_common(5)]
Counter(i for j in list_of_langs for i in j ).most_common(5 )<feature_engineering>
from keras.models import Sequential
from keras.layers import Dense , Dropout , Lambda, Flatten
from keras.optimizers import Adam ,RMSprop
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras.preprocessing.image import ImageDataGenerator
train['num_of_langs'] = train['spoken_languages'].apply(lambda x: len(x)if x!= {} else 0)
 test['num_of_langs'] = test['spoken_languages'].apply(lambda x: len(x)if x!= {} else 0)
 train['all_langs'] = train['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name']for i in x]))
if x!= {} else '')
 test['all_langs'] = test['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x]))
if x!= {} else '')
 for l in top_langs:
train['lang_' + l] = train['all_langs'].apply(lambda x: 1 if l in x else 0)
 test['lang_'+ l] = test['all_langs'].apply(lambda x: 1 if l in x else 0 )<define_variables>
train = pd.read_csv(".. /input/train.csv")
 print(train.shape )
for i, e in enumerate(train['Keywords'][:2]):
print(i,e )<count_values>
test= pd.read_csv(".. /input/test.csv")
 print(test.shape )
list_of_keys = list(train['Keywords'].apply(lambda x: [i['name'] for i in x] if x!= {} else []))
Counter(i for j in list_of_keys for i in j ).most_common(10 )<feature_engineering>
X_train =(train.iloc[:,1:].values ).astype('float32')
 y_train = train.iloc[:,0].values.astype('int32')
 X_test = test.values.astype('float32' )
top_keywords = [m[0] for m in Counter(i for j in list_of_keys for i in j ).most_common(10)]
train['num_of_keywords'] = train['Keywords'].apply(lambda x: len(x)if x!={} else 0)
 test['num_of_keywords'] = test['Keywords'].apply(lambda x: len(x)if x!={} else 0)
 train['all_keywords'] = train['Keywords'].apply(lambda x: ' '.join(sorted([i['name']for i in x]))
if x!= {} else '')
 test['all_keywords'] = test['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x]))
if x!={} else '')
 for k in top_keywords:
train['keyword_'+ k] = train['all_keywords'].apply(lambda x: 1 if k in x else 0)
 test['keyword_'+ k] = test['all_keywords'].apply(lambda x: 1 if k in x else 0)
<define_variables>
mean_px = X_train.mean().astype(np.float32)
 std_px = X_train.std().astype(np.float32)
 def standardize(x):
return(x-mean_px)/std_px
for i, e in enumerate(train['cast'][:1]):
print(i,e )<count_values>
y_train= to_categorical(y_train)
 num_classes = y_train.shape[1]
num_classes
print('Number of casts used per movie:')
 train['cast'].apply(lambda x: len(x)if x!={} else 0 ).value_counts().head(10 )<define_variables>
seed = 10
np.random.seed(seed )
list_cast_name = list(train['cast'].apply(lambda x: [i['name'] for i in x]if x!= {} else []))
top_cast_name = [m[0] for m in Counter(i for j in list_cast_name for i in j ).most_common(20)]
Counter(i for j in list_cast_name for i in j ).most_common(20 )<feature_engineering>
from keras.models import Sequential
from keras.layers.core import Lambda , Dense, Flatten, Dropout
from keras.callbacks import EarlyStopping
from keras.layers import BatchNormalization, Convolution2D , MaxPooling2D
train['num_of_cast']= train['cast'].apply(lambda x: len(x)if x!={} else 0)
 test['num_of_cast'] = test['cast'].apply(lambda x: len(x)if x!={} else 0)
 train['all_cast_name'] = train['cast'].apply(lambda x: ' '.join(sorted([i['name']for i in x]))
if x!={} else '')
 test['all_cast_name'] = test['cast'].apply(lambda x: ' '.join(sorted([i['name']for i in x]))
if x!= {} else '')
 for c in top_cast_name:
train['cast_name_'+ c]= train['all_cast_name'].apply(lambda x: 1 if c in x else 0)
 test['cast_name_'+ c]= test['all_cast_name'].apply(lambda x: 1 if c in x else 0 )<count_values>
model= Sequential()
model.add(Lambda(standardize,input_shape=(28,28,1)))
 model.add(Flatten())
 model.add(Dense(10, activation='softmax'))
print("input shape ",model.input_shape)
 print("output shape ",model.output_shape )
print('Number of crew members per movie:')
 train['crew'].apply(lambda x: len(x)if x!= {} else 0 ).value_counts().head(10 )<count_values>
model.compile(optimizer=RMSprop(lr=0.001),
loss='categorical_crossentropy',
metrics=['accuracy'] )
list_crew_names = list(train['crew'].apply(lambda x: [i['name'] for i in x] if x!= {} else [] ).values)
 Counter(i for j in list_crew_names for i in j ).most_common(15 )<feature_engineering>
X = X_train
y = y_train
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.20, random_state=10)
 batches = gen.flow(X_train, y_train, batch_size=64)
 val_batches=gen.flow(X_val, y_val, batch_size=64 )
top_crew_names = [m[0] for m in Counter(i for j in list_crew_names for i in j ).most_common(20)]
train['num_of_crew'] = train['crew'].apply(lambda x: len(x)if x!= {} else 0)
 test['num_of_crew']= test['crew'].apply(lambda x: len(x)if x!= {} else 0)
 for cn in top_crew_names:
train['crew_name_'+ cn]= train['crew'].apply(lambda x: 1 if cn in str(x)else 0)
 test['crew_name_'+ cn] = test['crew'].apply(lambda x: 1 if cn in str(x)else 0 )<count_missing_values>
history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=3,
validation_data=val_batches, validation_steps=val_batches.n )
train['homepage'].isna().sum()<feature_engineering>
def get_cnn_model() :
model = Sequential([
Lambda(standardize, input_shape=(28,28,1)) ,
Convolution2D(32,(3,3), activation='relu'),
Convolution2D(32,(3,3), activation='relu'),
MaxPooling2D() ,
Convolution2D(64,(3,3), activation='relu'),
Convolution2D(64,(3,3), activation='relu'),
MaxPooling2D() ,
Flatten() ,
Dense(512, activation='relu'),
Dense(10, activation='softmax')
 ])
 model.compile(Adam() , loss='categorical_crossentropy',
metrics=['accuracy'])
 return model
train['has_homepage'] = 1
train.loc[pd.isnull(train['homepage']),"has_homepage"] = 0
test['has_homepage'] = 1
test.loc[pd.isnull(test['homepage']),"has_homepage"] = 0<count_missing_values>
model= get_cnn_model()
model.optimizer.lr=0.01
train['runtime'].isna().sum()<data_type_conversions>
history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=1,
validation_data=val_batches, validation_steps=val_batches.n )
train['runtime'].fillna(train['runtime'].mean() ,inplace= True )<correct_missing_values>
def get_bn_model() :
model = Sequential([
Lambda(standardize, input_shape=(28,28,1)) ,
Convolution2D(32,(3,3), activation='relu'),
BatchNormalization(axis=1),
Convolution2D(32,(3,3), activation='relu'),
MaxPooling2D() ,
BatchNormalization(axis=1),
Convolution2D(64,(3,3), activation='relu'),
BatchNormalization(axis=1),
Convolution2D(64,(3,3), activation='relu'),
MaxPooling2D() ,
Flatten() ,
BatchNormalization() ,
Dense(512, activation='relu'),
BatchNormalization() ,
Dense(10, activation='softmax')
 ])
 model.compile(Adam() , loss='categorical_crossentropy', metrics=['accuracy'])
 return model
test['runtime'].fillna(test['runtime'].mean() ,inplace= True )<feature_engineering>
model= get_bn_model()
model.optimizer.lr=0.01
history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=1,
validation_data=val_batches, validation_steps=val_batches.n )
train.loc[train['release_date'].isnull() == True, 'release_date'] = '01/01/98'
test.loc[test['release_date'].isnull() == True, 'release_date'] = '01/01/98' 
<categorify>
model.optimizer.lr=0.01
gen = image.ImageDataGenerator()
batches = gen.flow(X, y, batch_size=64)
 history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=3 )
<data_type_conversions><EOS> 
predictions = model.predict_classes(X_test, verbose=0)
 submissions=pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) ,
"Label": predictions})
 submissions.to_csv("Digit_Recog.csv", index=False, header=True )