Kaggle competition: TensorFlow 2.0 Question Answering
df_test = pd.read_csv('.. /input/quickdraw-doodle-recognition/test_simplified.csv')
 df_test['drawing'] = df_test['drawing'].apply(lambda x: json.loads(x))<predict_on_test>
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
nq_train_file = '.. /input/tensorflow2-question-answering/simplified-nq-train.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000
x_test = np.zeros(( df_test.shape[0], img_size, img_size, 1))
for i, drawing in enumerate(df_test['drawing'].values): x_test[i, :, :, 0] = draw2pixel(drawing)
 y_test = np.argsort(-model_mobilenet.predict(x_test, verbose=1)) [:, 0:3]
df_word = pd.DataFrame({'top 1': y_test[:, 0], 'top 2': y_test[:, 1], 'top 3': y_test[:, 2]})
 df_word = df_word.replace(classreverse_dict)
 df_word['submission'] = df_word['top 1'] + ' ' + df_word['top 2'] + ' ' + df_word['top 3']<save_to_csv>
if True:
importlib.reload(bert_utils )
out = pd.DataFrame({'key_id': df_test['key_id'], 'word': df_word['submission']})
 out.to_csv('submission.csv', index=False )<load_from_csv>
with open('.. /input/bert-joint-baseline/bert_config.json','r')as f:
config = json.load(f)
 print(json.dumps(config,indent=4))
def f2cat(filename: str)-> str:
return filename.split('.')[0]
class Simplified() :
def __init__(self, input_path='./input'):
self.input_path = input_path
def list_all_categories(self):
files = os.listdir(os.path.join(self.input_path, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower)
 def read_training_csv(self, category, nrows=None, usecols=None, drawing_transform=False):
df = pd.read_csv(os.path.join(self.input_path, 'train_simplified', category + '.csv'),
nrows=nrows, parse_dates=['timestamp'], usecols=usecols)
 if drawing_transform:
df['drawing'] = df['drawing'].apply(json.loads)
 return df<feature_engineering>
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
unique_id = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')
 input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [unique_id,input_ids,input_mask,segment_ids]
if input_ is not None],
[unique_id,start_logits,end_logits,ans_type],
name='bert-baseline' )
Total time {}s'.format(end,(end - start ).seconds))<define_variables>
small_config = config.copy()
small_config['vocab_size']=16
small_config['hidden_size']=64
small_config['max_position_embeddings'] = 32
small_config['num_hidden_layers'] = 4
small_config['num_attention_heads'] = 4
small_config['intermediate_size'] = 256
small_config
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
def split_train_val() :
ALL_FILES = glob.glob('.. /input/shuffle-csvs/*.csv.gz')
 VALIDATION_FILE = '.. /input/shuffle-csvs/train_k'+str(int(random.random() *93)) +'.csv.gz'
ALL_FILES.remove(VALIDATION_FILE)
 np.random.seed(seed=1987)
 return ALL_FILES, VALIDATION_FILE
def apk(actual, predicted, k=3):

 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower)
 def plot_batch(x):
cols = 4
rows = 6
fig, axs = plt.subplots(nrows=rows, ncols=cols, sharex=True, sharey=True, figsize=(18, 18))
for i in range(rows):
for k in range(0,3):
ax = axs[i, k]
ax.imshow(x[i, :, :, k], cmap=plt.cm.gray)
 ax.axis('off')
 ax = axs[i, 3]
ax.imshow(x[i, :, :],)
 ax.axis('off')
 fig.tight_layout()
plt.show() ;<define_search_space>
model= mk_model(config )
AUGMENTATION = True
STEPS = 200
BATCH_SIZE = 400
EPOCHS = 10
NCATS = 340
LEARNING_RATE = 0.002
IMG_SHAPE =(128,128,3)
 IMG_SIZE = IMG_SHAPE[0]<feature_engineering>
class DummyObject:
def __init__(self,**kwargs):
self.__dict__.update(kwargs)
 FLAGS=DummyObject(skip_nested_contexts=True,
max_position=50,
max_contexts=48,
max_query_length=64,
max_seq_length=512,
doc_stride=128,
include_unknowns=-1.0,
n_best_size=20,
max_answer_length=30 )
def image_generator(size, batchsize, lw=6, augmentation = False):
while True:
for filename in ALL_FILES:
for df in pd.read_csv(filename, chunksize=batchsize):
df['drawing'] = df['drawing'].apply(eval)
 x = np.zeros(( len(df), size, size,3))
for i, raw_strokes in enumerate(df.drawing.values):
x[i] = draw_cv2(raw_strokes, size=size, lw=lw, augmentation = augmentation)
 x = x / 255.
 x = x.reshape(( len(df), size, size, 3)).astype(np.float32)
 y = keras.utils.to_categorical(df.y, num_classes=NCATS)
 yield x, y
def valid_generator(valid_df, size, batchsize, lw=6):
while(True):
for i in range(0,len(valid_df),batchsize):
chunk = valid_df[i:i+batchsize]
x = np.zeros(( len(chunk), size, size,3))
for i, raw_strokes in enumerate(chunk.drawing.values):
x[i] = draw_cv2(raw_strokes, size=size, lw=lw)
 x = x / 255.
 x = x.reshape(( len(chunk), size, size,3)).astype(np.float32)
 y = keras.utils.to_categorical(chunk.y, num_classes=NCATS)
 yield x,y
def test_generator(test_df, size, batchsize, lw=6):
for i in range(0,len(test_df),batchsize):
chunk = test_df[i:i+batchsize]
x = np.zeros(( len(chunk), size, size,3))
for i, raw_strokes in enumerate(chunk.drawing.values):
x[i] = draw_cv2(raw_strokes, size=size, lw=lw)
 x = x / 255.
 x = x.reshape(( len(chunk), size, size, 3)).astype(np.float32)
 yield x
ALL_FILES, VALIDATION_FILE = split_train_val()
train_datagen = image_generator(size=IMG_SIZE, batchsize=BATCH_SIZE, augmentation = AUGMENTATION)
 valid_df = pd.read_csv(VALIDATION_FILE)
 valid_df['drawing'] = valid_df['drawing'].apply(eval)
 validation_steps = len(valid_df)//BATCH_SIZE
valid_datagen = valid_generator(valid_df, size=IMG_SIZE, batchsize=BATCH_SIZE )<import_modules>
eval_records = ".. /input/bert-joint-baseline/nq-test.tfrecords"
if on_kaggle_server and private_dataset:
eval_records='nq-test.tfrecords'
if not os.path.exists(eval_records):
eval_writer = bert_utils.FeatureWriter(
 filename=os.path.join(eval_records),
is_training=False)
 tokenizer = tokenization.FullTokenizer(vocab_file='.. /input/bert-joint-baseline/vocab-nq.txt',
do_lower_case=True)
 features = []
convert = bert_utils.ConvertExamples2Features(tokenizer=tokenizer,
is_training=False,
output_fn=eval_writer.process_feature,
collect_stat=False)
 n_examples = 0
tqdm_notebook= tqdm.tqdm_notebook if not on_kaggle_server else None
for examples in bert_utils.nq_examples_iter(input_file=nq_test_file,
is_training=False,
tqdm=tqdm_notebook):
for example in examples:
n_examples += convert(example)
 eval_writer.close()
print('number of test examples: %d, written to file: %d' %(n_examples,eval_writer.num_features))
def top_3_accuracy(y_true, y_pred):
return keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)
 reducer = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5)
 checkpointer = ModelCheckpoint(filepath='mobileNet_ckpt.hdf5', verbose=2, save_best_only=True)
 model = load_model('.. /input/mobilenetfile/mobileNet.hdf5', custom_objects = {'top_3_accuracy':top_3_accuracy})
 opt = Adam(lr = LEARNING_RATE)
 model.compile(optimizer = opt, loss = 'categorical_crossentropy', metrics = ['accuracy', top_3_accuracy])
 model.summary()<train_model>
seq_length = FLAGS.max_seq_length
name_to_features = {
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
}
def _decode_record(record, name_to_features=name_to_features):

 example = tf.io.parse_single_example(serialized=record, features=name_to_features)
 for name in list(example.keys()):
t = example[name]
if name != 'unique_id':
t = tf.cast(t, dtype=tf.int32)
 example[name] = t
return example
def _decode_tokens(record):
return tf.io.parse_single_example(serialized=record,
features={
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"token_map" : tf.io.FixedLenFeature([seq_length], tf.int64)
 })
history = model.fit_generator(train_datagen,
steps_per_epoch=STEPS,
epochs=EPOCHS,
verbose=2,
validation_data=valid_datagen,
validation_steps=validation_steps,
callbacks=[checkpointer,reducer])
 model.save('mobileNet.hdf5' )<save_to_csv>
raw_ds = tf.data.TFRecordDataset(eval_records)
 token_map_ds = raw_ds.map(_decode_tokens)
 decoded_ds = raw_ds.map(_decode_record)
 ds = decoded_ds.batch(batch_size=16,drop_remainder=False )
submission_df = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
submission_df['drawing'] = submission_df['drawing'].apply(eval)
 submission_datagen = test_generator(submission_df, size=IMG_SIZE, batchsize=BATCH_SIZE)
 submission_predictions = model.predict_generator(submission_datagen, math.ceil(len(submission_df)/BATCH_SIZE))
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3 = preds2catids(submission_predictions)
 top3cats = top3.replace(id2cat)
 submission_df['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = submission_df[['key_id', 'word']]
submission.to_csv('submission.csv', index=False )<set_options>
result=model.predict_generator(ds,verbose=1 )
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
start = dt.datetime.now()<import_modules>
np.savez_compressed('bert-joint-baseline-output.npz',
**dict(zip(['uniqe_id','start_logits','end_logits','answer_type_logits'],
result)) )
from tensorflow.keras.models import load_model
import h5py<load_pretrained>
Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx"] )
model = load_model('.. /input/model-pretrained/model.h5' )<define_variables>
class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None
DP_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
NCSVS = 100
NCATS = 340
np.random.seed(seed=1987)
 tf.set_random_seed(seed=1987)
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower )<compute_test_metric>
class EvalExample(object):

 def __init__(self, example_id, candidates):
self.example_id = example_id
self.candidates = candidates
self.results = {}
self.features = {}
def apk(actual, predicted, k=3):

 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )<define_variables>
def get_best_indexes(logits, n_best_size):

 index_and_score = sorted(
 enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
 best_indexes = []
for i in range(len(index_and_score)) :
if i >= n_best_size:
break
best_indexes.append(index_and_score[i][0])
 return best_indexes
def top_k_indices(logits,n_best_size,token_map):
indices = np.argsort(logits[1:])+1
indices = indices[token_map[indices]!=-1]
return indices[-n_best_size:]
STEPS = 800
EPOCHS = 16
size = 64
batchsize = 680<choose_model_class>
def compute_predictions(example):

 predictions = []
n_best_size = FLAGS.n_best_size
max_answer_length = FLAGS.max_answer_length
i = 0
for unique_id, result in example.results.items() :
if unique_id not in example.features:
raise ValueError("No feature found with unique_id:", unique_id)
 token_map = np.array(example.features[unique_id]["token_map"])
 start_indexes = top_k_indices(result.start_logits,n_best_size,token_map)
 if len(start_indexes)==0:
continue
end_indexes = top_k_indices(result.end_logits,n_best_size,token_map)
 if len(end_indexes)==0:
continue
indexes = np.array(list(np.broadcast(start_indexes[None],end_indexes[:,None])))
 indexes = indexes[(indexes[:,0]<indexes[:,1])*(indexes[:,1]-indexes[:,0]<max_answer_length)]
for start_index,end_index in indexes:
summary = ScoreSummary()
summary.short_span_score =(
 result.start_logits[start_index] +
result.end_logits[end_index])
 summary.cls_token_score =(
 result.start_logits[0] + result.end_logits[0])
 summary.answer_type_logits = result.answer_type_logits-result.answer_type_logits.mean()
start_span = token_map[start_index]
end_span = token_map[end_index] + 1
score = summary.short_span_score - summary.cls_token_score
predictions.append(( score, i, summary, start_span, end_span))
i += 1
score = -10000.0
short_span = Span(-1, -1)
 long_span = Span(-1, -1)
 summary = ScoreSummary()
if predictions:
score, _, summary, start_span, end_span = sorted(predictions, reverse=True)[0]
short_span = Span(start_span, end_span)
 for c in example.candidates:
start = short_span.start_token_idx
end = short_span.end_token_idx
if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
long_span = Span(c["start_token"], c["end_token"])
 break
summary.predicted_label = {
"example_id": int(example.example_id),
"long_answer": {
"start_token": int(long_span.start_token_idx),
"end_token": int(long_span.end_token_idx),
"start_byte": -1,
"end_byte": -1
},
"long_answer_score": float(score),
"short_answers": [{
"start_token": int(short_span.start_token_idx),
"end_token": int(short_span.end_token_idx),
"start_byte": -1,
"end_byte": -1
}],
"short_answer_score": float(score),
"yes_no_answer": "NONE",
"answer_type_logits": summary.answer_type_logits.tolist() ,
"answer_type": int(np.argmax(summary.answer_type_logits))
}
return summary
model = MobileNet(input_shape=(size, size, 1), alpha=1., weights=None, classes=NCATS)
 model.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])
 print(model.summary() )<load_pretrained>
def read_candidates_from_one_split(input_path):

 candidates_dict = {}
print("Reading examples from: %s" % input_path)
 if input_path.endswith(".gz"):
with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, "rb")) as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
else:
with tf.io.gfile.GFile(input_path, "r")as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
return candidates_dict
model.load_weights('.. /input/model-pretrained/model.h5' )<load_from_csv>
def read_candidates(input_pattern):

 input_paths = tf.io.gfile.glob(input_pattern)
 final_dict = {}
for input_path in input_paths:
final_dict.update(read_candidates_from_one_split(input_path))
return final_dict
valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)) , nrows=34000)
 x_valid = df_to_image_array_xd(valid_df, size)
 y_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3))<drop_column>
all_results = [bert_utils.RawResult(*x)for x in zip(*result)]
print("Going to candidates file")
 candidates_dict = read_candidates('.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl')
 print("setting up eval features")
 eval_features = list(token_map_ds)
 print("compute_pred_dict")
 tqdm_notebook= tqdm.tqdm_notebook
nq_pred_dict = compute_pred_dict(candidates_dict,
eval_features,
all_results,
tqdm=tqdm_notebook)
 predictions_json = {"predictions": list(nq_pred_dict.values())}
print("writing json")
 with tf.io.gfile.GFile('predictions.json', "w")as f:
json.dump(predictions_json, f, indent=4 )
train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))<prepare_x_and_y>
def create_short_answer(entry):
answer = []
if entry['answer_type'] == 0:
return ""
elif entry['answer_type'] == 1:
return 'YES'
elif entry['answer_type'] == 2:
return 'NO'
elif entry["short_answer_score"] < 1.5:
return ""
else:
for short_answer in entry["short_answers"]:
if short_answer["start_token"] > -1:
answer.append(str(short_answer["start_token"])+ ":" + str(short_answer["end_token"]))
return " ".join(answer)
 def create_long_answer(entry):
answer = []
if entry['answer_type'] == 0:
return ''
elif entry["long_answer_score"] < 1.5:
return ""
elif entry["long_answer"]["start_token"] > -1:
answer.append(str(entry["long_answer"]["start_token"])+ ":" + str(entry["long_answer"]["end_token"]))
return " ".join(answer )
%%timeit
x, y = next(train_datagen )<predict_on_test>
test_answers_df = pd.read_json(".. /working/predictions.json")
 for var_name in ['long_answer_score','short_answer_score','answer_type']:
test_answers_df[var_name] = test_answers_df['predictions'].apply(lambda q: q[var_name])
 test_answers_df["long_answer"] = test_answers_df["predictions"].apply(create_long_answer)
 test_answers_df["short_answer"] = test_answers_df["predictions"].apply(create_short_answer)
 test_answers_df["example_id"] = test_answers_df["predictions"].apply(lambda q: str(q["example_id"]))
long_answers = dict(zip(test_answers_df["example_id"], test_answers_df["long_answer"]))
short_answers = dict(zip(test_answers_df["example_id"], test_answers_df["short_answer"]))
test_answers_df.head()
valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('Map3: {:.3f}'.format(map3))<load_from_csv>
sample_submission = pd.read_csv(".. /input/tensorflow2-question-answering/sample_submission.csv")
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains("_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains("_short"), "PredictionString"] = short_prediction_strings
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test.head()
x_test = df_to_image_array_xd(test, size)
 print(test.shape, x_test.shape)
 print('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3))<predict_on_test>
sample_submission.to_csv('submission.csv', index=False )
test_predictions = model.predict(x_test, batch_size=128, verbose=1)
 top3 = preds2catids(test_predictions)
 top3.head()
top3.shape
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 top3cats.head()
top3cats.shape<save_to_csv>
yes_answers = sample_submission[sample_submission['PredictionString'] == 'YES']
yes_answers
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('gs_mn_submission_{}.csv'.format(int(map3 * 10**4)) , index=False)
 submission.head()
submission.shape<train_model>
no_answers = sample_submission[sample_submission['PredictionString'] == 'NO']
no_answers
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))<set_options>
blank_answers = sample_submission[sample_submission['PredictionString'] == '']
blank_answers.head()
<define_variables><EOS> 
blank_answers.count()