Kaggle competition: Dogs vs. Cats Redux: Kernels Edition
<SOS> metric: LogLoss Kaggle data source: dogs-vs.-cats-redux:-kernels-edition<predict_on_test>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
import numpy as np
from sklearn.metrics import confusion_matrix
import os, random
from shutil import copy, copytree
import gc
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )<install_modules>
from fastai.conv_learner import*
!pip install transformers<import_modules>
train_path = '.. /input/train/'
test_path = '.. /input/test/'
train_samples = 10000
valid_samples = 2500
from tqdm import tqdm<load_from_csv>
train_dog = [train_path+i for i in os.listdir(train_path)if 'dog' in i]
train_cat = [train_path+i for i in os.listdir(train_path)if 'cat' in i]
df = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 df.head()<define_variables>
random.shuffle(train_dog)
 random.shuffle(train_cat )
sentences = df.text.values
labels = df.target.values<load_pretrained>
train_dog_images = train_dog[:train_samples]
train_cat_images = train_cat[:train_samples]
valid_dog_images = train_dog[- valid_samples:]
valid_cat_images = train_cat[- valid_samples:]
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',do_lower_case = True )<categorify>
for i in range(0,train_samples):
shutil.copy(train_dog_images[i], '.. /working/dogcats/train/dog/')
 shutil.copy(train_cat_images[i], '.. /working/dogcats/train/cat/' )
input_ids = []
for sent in tqdm(sentences):
encoded_sent = tokenizer.encode(
 sent,
add_special_tokens = True, 
)
 input_ids.append(encoded_sent)
 print('Original: ', sentences[0])
 print('Token IDs:', input_ids[0] )<categorify>
for i in range(0,valid_samples):
shutil.copy(valid_dog_images[i], '.. /working/dogcats/valid/dog/')
 shutil.copy(valid_cat_images[i], '.. /working/dogcats/valid/cat/' )
MAX_LEN = 64
print('
Padding/truncating all sentences to %d values...' % MAX_LEN)
 print('
Padding token: "{:}", ID: {:}'.format(tokenizer.pad_token, tokenizer.pad_token_id))
input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype="long",
value=0, truncating="post", padding="post")
 print('
Done.' )<define_variables>
shutil.copytree(test_path, '.. /working/dogcats/test/' )
attention_masks = []
for sent in tqdm(input_ids):
att_mask = [int(token_id > 0)for token_id in sent]
attention_masks.append(att_mask )<split>
path = '.. /working/dogcats/'
image_size = 224
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=2018, test_size=0.1)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,
random_state=2018, test_size=0.1 )<data_type_conversions>
torch.cuda.is_available()
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels)
 validation_labels = torch.tensor(validation_labels)
 train_masks = torch.tensor(train_masks)
 validation_masks = torch.tensor(validation_masks )<create_dataframe>
torch.backends.cudnn.enabled
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size )<load_pretrained>
arch = resnet34
tfms = tfms_from_model(arch, image_size, aug_tfms=transforms_side_on, max_zoom=1.1)
 data = ImageClassifierData.from_paths(path, tfms=tfms, test_name='test')
 learn = ConvLearner.pretrained(arch, data, precompute=True )
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()<train_model>
lrf = learn.lr_find()
params = list(model.named_parameters())
 print('The BERT model has {:} different named parameters.
 '.format(len(params)))
 print('==== Embedding Layer ====
')
 for p in params[0:5]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== First Transformer ====
')
 for p in params[5:21]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== Output Layer ====
')
 for p in params[-4:]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))<choose_model_class>
learn.fit(0.01, 2 )
optimizer = AdamW(model.parameters() ,
lr = 1e-5,
eps = 1e-8
)<compute_test_metric>
gc.collect()
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )<train_model>
learn.precompute = False
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 40 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(train_dataloader)
 loss_values.append(avg_train_loss)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print(" Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
print(" Validation took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )<load_from_csv>
learn.fit(0.01, n_cycle=3, cycle_len=1 )
df = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 print('Number of test sentences: {:,}
'.format(df.shape[0]))
sentences = df.text.values
input_ids = []
for sent in tqdm(sentences):
encoded_sent = tokenizer.encode(
 sent,
add_special_tokens = True, 
)
 input_ids.append(encoded_sent)
 input_ids = pad_sequences(input_ids, maxlen=MAX_LEN,
dtype="long", truncating="post", padding="post")
 attention_masks = []
for seq in tqdm(input_ids):
seq_mask = [float(i>0)for i in seq]
attention_masks.append(seq_mask)
 prediction_inputs = torch.tensor(input_ids)
 prediction_masks = torch.tensor(attention_masks)
 batch_size = 32
prediction_data = TensorDataset(prediction_inputs, prediction_masks)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)
<predict_on_test>
gc.collect()
print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs)))
 model.eval()
predictions=[]
for batch in tqdm(prediction_dataloader):
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 print(' DONE.' )<define_variables>
learn.unfreeze()
answers = []
for i in tqdm(range(len(predictions))):
for j in range(len(predictions[i])) :
answers.append(softmax(predictions[i][j]))
final_answers = []
for i in range(len(answers)) :
final_answers.append(np.argmax(answers[i]))<load_from_csv>
lr=np.array([1e-4,1e-3,1e-2])
 learn.fit(lr, 3, cycle_len=1, cycle_mult=2 )
sample = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv' )<feature_engineering>
gc.collect()
sample['target'] = final_answers<save_to_csv>
valid_log_predictions,y = learn.TTA()
valid_prob_predictions = np.mean(np.exp(valid_log_predictions),0 )
sample.to_csv('samplesubmission.csv',index = False)
 sample.head()<set_options>
accuracy_np(valid_prob_predictions, y )
%matplotlib inline<set_options>
valid_predictions = np.argmax(valid_prob_predictions, axis=1 )
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))<load_from_csv>
log_predictions,_ = learn.TTA(is_test=True )
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv',index_col=0)
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv',index_col=0)
 print(train.shape)
 train.head()<concatenate>
prob_predictions = np.mean(np.exp(log_predictions),0)
 prob_predictions = prob_predictions[:,1]
df_all = pd.concat([train[train.columns[:-1]],test])
 df_all.head(5 )<prepare_x_and_y>
submission = pd.DataFrame({'id':os.listdir(f'{path}test'), 'label':prob_predictions} )
y = train.SalePrice<prepare_output>
submission['id'] = submission['id'].map(lambda x: x.split('.')[0])
 submission['id'] = submission['id'].astype(int)
 submission = submission.sort_values('id' )
y_original = y.copy()
y = np.log(y )<drop_column>
submission.to_csv("submission.csv", index = False )