Kaggle competition: Tweet Sentiment Extraction
def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(seed=42 )<import_modules>
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 print('Training data shape: ', train.shape)
 print('Testing data shape: ', test.shape )
from efficientnet_pytorch import EfficientNet<load_pretrained>
train.isnull().sum()
class EfficientNetB3(nn.Module):
def __init__(self, pretrained):
super(EfficientNetB3, self ).__init__()
if pretrained == True:
self.model = EfficientNet.from_name('efficientnet-b3')
 self.model.load_state_dict(torch.load('.. /input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth'))
else:
self.model = EfficientNet.from_pretrained(None)
 in_features = self.model._fc.in_features
self.l0 = nn.Linear(in_features, config.CLASSES)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.extract_features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 return l0
class EfficientNetB0(nn.Module):
def __init__(self, pretrained):
super(EfficientNetB0, self ).__init__()
if pretrained == True:
self.model = EfficientNet.from_name('efficientnet-b0')
 self.model.load_state_dict(torch.load('.. /input/efficientnet-pytorch/efficientnet-b0-08094119.pth'))
else:
self.model = EfficientNet.from_pretrained(None)
 in_features = self.model._fc.in_features
self.l0 = nn.Linear(in_features, config.CLASSES)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.extract_features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 return l0<categorify>
test.isnull().sum()
def get_tiles(img, mode=0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 255 ).sum()
if len(img3)< n_tiles:
img3 = np.pad(img3,[[0,n_tiles-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info >= n_tiles<save_to_csv>
train.loc[train['text'].isnull() ]
def write_image(path, img):
img = cv2.convertScaleAbs(img, alpha=(255.0))
cv2.imwrite(path, cv2.cvtColor(img, cv2.COLOR_RGB2BGR))<define_variables>
train.dropna(axis = 0, how ='any',inplace=True)
 train.reset_index(inplace=True)
 train.shape
tile_size = 256
image_size = 256
n_tiles = 16
batch_size = 2<create_dataframe>
positive_text = train[train['sentiment'] == 'positive']['selected_text']
negative_text = train[train['sentiment'] == 'negative']['selected_text']
neutral_text = train[train['sentiment'] == 'neutral']['selected_text']
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False, 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.aug = albumentations.Compose([
albumentations.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], always_apply=True)
 ])
 def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(data_dir, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, OK = get_tiles(image, self.tile_mode)
 if self.rand:
idxes = np.random.choice(list(range(self.n_tiles)) , self.n_tiles, replace=False)
 else:
idxes = list(range(self.n_tiles))
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 img = images
img = np.transpose(img,(1, 2, 0))
img = 1 - img
img = cv2.resize(img,(768, 768))
write_image(f'{img_id}.png', img)
 img = skimage.io.MultiImage(f'{img_id}.png')[-1]
img = Image.fromarray(img ).convert("RGB")
 img = self.aug(image=np.array(img)) ["image"]
img = np.transpose(img,(2, 0, 1)).astype(np.float32)
 return { 'image': torch.tensor(img, dtype=torch.float)}<define_variables>
def clean_text(text):

 text = text.lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 '(new lines)
 text = re.sub('\w*\d\w*', '', text)
 return text
def text_preprocessing(text):

 tokenizer = nltk.tokenize.RegexpTokenizer(r'\w+')
 nopunc = clean_text(text)
 tokenized_text = tokenizer.tokenize(nopunc)
 remove_stopwords = [w for w in tokenized_text if w not in stopwords.words('english')]
combined_text = ' '.join(tokenized_text)
 return combined_text
ENSEMBLES = [
{
'model_name': 'efficientnet-b0',
'model_weight': 'efficientnetB0_0.pth',
'ensemble_weight': 1
}
]<load_pretrained>
positive_text_clean = positive_text.apply(lambda x: text_preprocessing(x))
negative_text_clean = negative_text.apply(lambda x: text_preprocessing(x))
neutral_text_clean = neutral_text.apply(lambda x: text_preprocessing(x))
device = config.device
models = []
for ensemble in ENSEMBLES:
model = EfficientNetB3(pretrained=True)
 model.load_state_dict(torch.load(ensemble['model_weight'], map_location=device))
model.to(device)
 models.append(model )<prepare_output>
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True
)
model.eval()
predictions = []
if check_for_images_dir() :
test_dataset = PANDADataset(
 df=test,
image_size=image_size,
n_tiles=n_tiles,
tile_mode=0 
)
 test_data_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=config.TEST_BATCH_SIZE,
shuffle=False, 
)
 for model in models:
preds = []
for idx, d in tqdm(enumerate(test_data_loader), total=len(test_data_loader)) :
inputs = d["image"]
inputs = inputs.to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.to('cpu' ).numpy())
 predictions.append(np.concatenate(preds))
predictions = np.mean(predictions, axis=0)
 predictions = predictions.argmax(1)
<save_to_csv>
MAX_LEN = 0
for text in train['text']:
input_ids = tokenizer.encode(text, add_special_tokens=True)
 MAX_LEN = max(MAX_LEN, len(input_ids))
print('Max length: ', MAX_LEN )
if config.DEBUG:
def quadratic_weighted_kappa(y_hat, y):
return cohen_kappa_score(y_hat, y, weights='quadratic')
 count = 0
for index, val in enumerate(test.isup_grade.values):
if predictions[index] == val:
count += 1
print(f"Accuracy Train is {(count / test.shape[0])* 100}")
 print(f'Kappa Train is {quadratic_weighted_kappa(predictions, test.isup_grade.values)}')
 else:
if len(predictions)> 0:
submission.isup_grade = predictions
submission.isup_grade = submission['isup_grade'].astype(int)
 submission.to_csv('submission.csv',index=False)
 print(submission.head() )<define_variables>
sent = ['positive','negative','neutral']
sent_id =[]
for i in sent:
sent_id.append(tokenizer.encode(i ).ids[0])
 sentiment_id = dict(zip(sent,sent_id))
sentiment_id
sys.path = [
'.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',
] + sys.path<import_modules>
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ':
chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
import numpy as np
import pandas as pd
import torch.nn.functional as F
import os
import torch
import torch.nn as nn
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms,models
from tqdm import tqdm_notebook as tqdm
import math
import torch.utils.model_zoo as model_zoo
import cv2
from sklearn.metrics import cohen_kappa_score
import openslide
import skimage.io
import random
from sklearn.metrics import cohen_kappa_score
import albumentations
from PIL import Image<set_options>
ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
class config:
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 IMG_WIDTH = 512
IMG_HEIGHT = 512
TEST_BATCH_SIZE = 16
CLASSES = 6
DEBUG = False<load_from_csv>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)

x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model
BASE_PATH = '.. /input/prostate-cancer-grade-assessment'
data_dir = f'{BASE_PATH}/test_images'
test = pd.read_csv(f'{BASE_PATH}/test.csv')
 submission = pd.read_csv(f'{BASE_PATH}/sample_submission.csv')
 if config.DEBUG:
data_dir = f'{BASE_PATH}/train_images'
test = pd.read_csv(f'{BASE_PATH}/train.csv' ).head(200 )<set_options>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(seed=42 )<import_modules>
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=5, batch_size=32, verbose=DISPLAY, callbacks=[sv],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i.h5'%(VER,fold))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
from efficientnet_pytorch import EfficientNet<load_pretrained>
result= []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 result.append(st)
 result[:5]
class EfficientNetB3(nn.Module):
def __init__(self, pretrained):
super(EfficientNetB3, self ).__init__()
if pretrained == True:
self.model = EfficientNet.from_name('efficientnet-b3')
 self.model.load_state_dict(torch.load('.. /input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth'))
else:
self.model = EfficientNet.from_pretrained(None)
 in_features = self.model._fc.in_features
self.l0 = nn.Linear(in_features, config.CLASSES)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.extract_features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 return l0
class EfficientNetB0(nn.Module):
def __init__(self, pretrained):
super(EfficientNetB0, self ).__init__()
if pretrained == True:
self.model = EfficientNet.from_name('efficientnet-b0')
 self.model.load_state_dict(torch.load('.. /input/efficientnet-pytorch/efficientnet-b0-08094119.pth'))
else:
self.model = EfficientNet.from_pretrained(None)
 in_features = self.model._fc.in_features
self.l0 = nn.Linear(in_features, config.CLASSES)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.extract_features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 return l0<categorify>
test['selected_text'] = result
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.read_csv('submission.csv' ).head(10 )