Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<randomize_order>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
emb_mean, emb_std = -0.005838499, 0.48782197
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8")as f:
for line in tqdm(f):
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= nb_words:
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
emb_mean, emb_std = -0.0033469985, 0.109855495
nb_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
with open(EMBEDDING_FILE, 'r')as f:
for line in tqdm(f):
if len(line)<= 100:
continue
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= nb_words:
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
emb_mean, emb_std = -0.0053247833, 0.49346462
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8", errors='ignore')as f:
for line in tqdm(f):
if len(line)<= 100:
continue
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= nb_words:
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix<normalization>
from fastai import *
from fastai.vision import *
import pandas as pd
import matplotlib.pyplot as plt
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_fasttext(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix123 = np.concatenate([embedding_matrix_1, embedding_matrix_2, embedding_matrix_3], axis = 1)
 embedding_matrix12 = np.concatenate([embedding_matrix_1, embedding_matrix_2], axis = 1)
 embedding_matrix13 = np.concatenate([embedding_matrix_1, embedding_matrix_3], axis = 1)
 embedding_matrix23 = np.concatenate([embedding_matrix_2, embedding_matrix_3], axis = 1)
 embedding_matrixs = [embedding_matrix12, embedding_matrix13, embedding_matrix23, embedding_matrix13]<normalization>
print('Make sure cudnn is enabled:', torch.backends.cudnn.enabled )
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 999
seed_everything(SEED )
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix):
super(NeuralNet, self ).__init__()
hidden_size = 128
self.embedding = nn.Embedding(max_features, embed_size*2)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.15)
 self.lstm = nn.LSTM(embed_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size*2, maxlen)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.linear = nn.Linear(1024, 64)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.15)
 self.out = nn.Linear(64, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<choose_model_class>
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 df.head(10 )
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, factor=0.6, min_lr=1e-4, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, torch.optim.Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
self.last_loss = np.inf
self.min_lr = min_lr
self.factor = factor
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def step(self, loss):
if loss > self.last_loss:
self.base_lrs = [max(lr * self.factor, self.min_lr)for lr in self.base_lrs]
self.max_lrs = [max(lr * self.factor, self.min_lr)for lr in self.max_lrs]
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs<compute_test_metric>
im = Image.open(df['path'][1])
 width, height = im.size
print(width,height)
 im.show()
def sigmoid(x):
return 1 /(1 + np.exp(-x))<compute_test_metric>
bs = 64
sz=224
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<split>
data.show_batch(rows=3, figsize=(7,6))
def train_and_predict1() :
seed_torch(SEED)
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 train_x0, valid_x0, train_y0, valid_y0 = train_test_split(train_X, train_y, test_size=0.05, shuffle=True, random_state=43)
 valid_preds = np.zeros(( len(valid_y0)))
 test_preds = np.zeros(( len(test_X)))
 epochs=3
for i in range(epochs):
x_train_fold = torch.tensor(train_x0, dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y0.reshape(len(train_y0), 1), dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(valid_x0, dtype=torch.long ).cuda()
y_val_fold = torch.tensor(valid_y0.reshape(len(valid_y0), 1), dtype=torch.float32 ).cuda()
model = NeuralNet(embedding_matrixs[i])
 model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction="mean")
 optimizer = torch.optim.Adam(model.parameters())
 step_size = 300
scheduler = CyclicLR(optimizer, base_lr=0.001, max_lr=0.002,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 train_epochs = 4
for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))
search_result = threshold_search(valid_y0, valid_preds_fold)
 print(search_result)
 for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
valid_preds += valid_preds_fold / epochs
test_preds += test_preds_fold / epochs
return valid_y0, valid_preds, test_preds<split>
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )
def train_and_predict2() :
seed_torch(SEED)
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 test_preds = np.zeros(( len(test_X)))
 epochs=3
avgThreshold = 0
for i in range(epochs):
train_x0, valid_x0, train_y0, valid_y0 = train_test_split(train_X, train_y, test_size=0.05, shuffle=True, random_state=43 * i)
 valid_preds = np.zeros(( len(valid_y0)))
 x_train_fold = torch.tensor(train_x0, dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y0.reshape(len(train_y0), 1), dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(valid_x0, dtype=torch.long ).cuda()
y_val_fold = torch.tensor(valid_y0.reshape(len(valid_y0), 1), dtype=torch.float32 ).cuda()
model = NeuralNet(embedding_matrixs[i])
 model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction="mean")
 optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 train_epochs = 4
for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))
search_result = threshold_search(valid_y0, valid_preds_fold)
 if epoch == train_epochs - 1:
avgThreshold += search_result['threshold'] / epochs
print(search_result)
 for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
test_preds += test_preds_fold / epochs
return avgThreshold, test_preds<split>
learn = cnn_learner(data, base_arch=models.resnet50, metrics = [quadratic_kappa] )
def train_and_predict3() :
seed_torch(SEED)
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 test_preds = np.zeros(( len(test_X)))
 splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED ).split(train_X, train_y))
train_preds = np.zeros(( len(train_X)))
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet(embedding_matrixs[i])
 model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction="mean")
 optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 train_epochs = 4
for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=False):
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))
search_result = threshold_search(train_y[valid_idx], valid_preds_fold)
 print(search_result)
 for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 return train_preds, test_preds<save_to_csv>
learn.fit_one_cycle(4,max_lr = 1e-2 )
<save_to_csv>
learn.fit_one_cycle(6, max_lr=slice(1e-6,1e-3))
<save_to_csv>
learn.export()
learn.save('stage-2' )
train_preds, test_preds = train_and_predict3()
search_result = threshold_search(train_y, train_preds)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction =(test_preds > search_result['threshold'] ).astype(int)
 sub.to_csv("submission.csv", index=False )<set_options>
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )
color = sns.color_palette()
%matplotlib inline
py.init_notebook_mode(connected=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999
print(os.listdir(".. /input")) 
<load_pretrained>
valid_preds = learn.get_preds(ds_type=DatasetType.Valid )
nlp = spacy.load('en_core_web_sm')
 doc = nlp(u'A few days later, Abigail complained that Elizabeth was pinching her and tearing at her bowels')
 for token in doc:
print(token.text, token.pos_, token.tag_, token.dep_ )<string_transform>
import numpy as np
import pandas as pd
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
import json
def word_locate(sentence, location):
count_words = 0
count_chars = 2
for word in sentence.split() :
count_words += 1
if location == count_chars:
return word, count_words
count_chars += len(word)
 count_chars += 1<string_transform>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def curr_prev_sentence(sentence, loc):
current_sentence = ""
prev_sentence = ""
detect = 0
count = 0
for char in sentence:
count += 1
current_sentence += char
if char == "." and detect == 0:
prev_sentence = current_sentence
current_sentence = ""
if char == "." and detect == 1:
return current_sentence, prev_sentence
if count == loc:
detect = 1<filter>
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1] )
def find_subject(sentence):
doc = nlp(sentence)
 for token in doc:
if token.dep_ == "nsubj":
return token.text
return "none"<define_variables>
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()
text1 = "After a few years of almost no work -- although he was a guest star on Lou Grant and Charlie's Angels in the late 1970s, he once summed up the 1970s as 'I cried and did a lot of gardening' -- he was hired in 1979 for his best-known role, self-made millionaire Palmer Cortlandt on ABC's long-running soap opera All My Children.Initially hired for only one year, he remained on contract through 2009.For much of his first decade on the show, Palmer was a ruthless villain, totally possessive of his daughter, Nina and violently threatening his ex-wife Daisy with being attacked by dobermans when she came back from the dead."<define_variables>
learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))
text2 = "When onlookers expressed doubt, claiming that the Proctor family was well regarded in the community, the girl promptly came out of her trance and told them it was all for 'sport'.On March 29, 1692, Abigail Williams and Mercy Lewis again said they were being tormented by Elizabeth's spectre.A few days later, Abigail complained that Elizabeth was pinching her and tearing at her bowels, and said she saw Elizabeth's spectre as well as John's."<feature_engineering>
preds,y = learn.TTA(ds_type=DatasetType.Test )
word, loc = word_locate(text2, 360)
 print(word, " ", loc )<load_from_csv>
test_predictions = optR.predict(preds, coefficients )
with open('.. /input/gap-other-training-data/gap-training.tsv')as tsvfile:
reader = csv.DictReader(tsvfile, dialect='excel-tab')
 train_X = []
train_y = []
tr_label_A_list = []
tr_label_B_list = []
for row in reader:
text = row['Text']
text = text.lower()
new_textA = text
labelA = 0
labelB = 0
check = 0
for char_idx in range(len(text)) :
if char_idx == int(row['A-offset']):
new_textA = new_textA[:char_idx+check] + "person_loc " + new_textA[char_idx+check:]
check = 11
if char_idx == int(row['Pronoun-offset']):
new_textA = new_textA[:char_idx+check] + "pronom_loc " + new_textA[char_idx+check:]
check = 11
if row['A-coref'] == 'TRUE':
labelA = 1
train_X.append(new_textA)
 train_y.append(labelA)
 new_textB = text
label = 0
check = 0
for char_idx in range(len(text)) :
if char_idx == int(row['B-offset']):
new_textB = new_textB[:char_idx+check] + "person_loc " + new_textB[char_idx+check:]
check = 11
if char_idx == int(row['Pronoun-offset']):
new_textB = new_textB[:char_idx+check] + "pronom_loc " + new_textB[char_idx+check:]
check = 11
if row['B-coref'] == 'TRUE':
labelB = 1
train_X.append(new_textB)
 train_y.append(labelB)
 tr_label_A_list.append(labelA)
 tr_label_B_list.append(labelB )<load_from_csv>
sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()
with open('.. /input/gap-other-training-data/gap-training.tsv')as tsvfile:
reader = csv.DictReader(tsvfile, dialect='excel-tab')
 testA_X = []
testA_y = []
testB_X = []
testB_y = []
label_A_list = []
label_B_list = []
for row in reader:
text = row['Text']
text = text.lower()
new_textA = text
labelA = 0
labelB = 0
check = 0
for char_idx in range(len(text)) :
if char_idx == int(row['A-offset']):
new_textA = new_textA[:char_idx+check] + "person_loc " + new_textA[char_idx+check:]
check = 11
if char_idx == int(row['Pronoun-offset']):
new_textA = new_textA[:char_idx+check] + "pronom_loc " + new_textA[char_idx+check:]
check = 11
if row['A-coref'] == 'TRUE':
labelA = 1
testA_X.append(new_textA)
 testA_y.append(labelA)
 new_textB = text
label = 0
check = 0
for char_idx in range(len(text)) :
if char_idx == int(row['B-offset']):
new_textB = new_textB[:char_idx+check] + "person_loc " + new_textB[char_idx+check:]
check = 11
if char_idx == int(row['Pronoun-offset']):
new_textB = new_textB[:char_idx+check] + "pronom_loc " + new_textB[char_idx+check:]
check = 11
if row['B-coref'] == 'TRUE':
labelB = 1
testB_X.append(new_textB)
 testB_y.append(labelB)
 label_A_list.append(labelA)
 label_B_list.append(labelB )<define_variables>
sample_df.to_csv('submission.csv',index=False )