Kaggle competition: Tweet Sentiment Extraction
def get_tiles(img, mode=0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 255 ).sum()
if len(img3)< n_tiles:
img3 = np.pad(img3,[[0,n_tiles-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info >= n_tiles<save_to_csv>
train_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 test_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 sub_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv')
 train_df=train_df[train_df['sentiment'] != 'neutral']
print(train_df.info())
 train_df['len1'] = train_df['selected_text'].apply(lambda x:len(str(x ).split(' ')))
 print(train_df.sentiment.value_counts())
 train_df=train_df[train_df.len1<=18]
print(train_df.info())
 print(train_df.sentiment.value_counts())
 train_df=train_df.drop(['len1'],axis=1)
def write_image(path, img):
img = cv2.convertScaleAbs(img, alpha=(255.0))
cv2.imwrite(path, cv2.cvtColor(img, cv2.COLOR_RGB2BGR))<define_variables>
def rev_sentence(sentence):
words = sentence.split(' ')
 reverse_sentence = ' '.join(reversed(words))
return reverse_sentence
train = np.array(train_df)
 test = np.array(test_df)
 !mkdir -p data
use_cuda = True
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
def do_qa_train(train):
output = {}
output['version'] = 'v1.0'
output['data'] = []
paragraphs = []
for line in train:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer.lower() })
 break
qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
tile_size = 256
image_size = 256
n_tiles = 16
batch_size = 2<create_dataframe>
qa_train = do_qa_train(train)
 with open('data/train.json', 'w')as outfile:
json.dump(qa_train, outfile)
 output = {}
output['version'] = 'v1.0'
output['data'] = []
def do_qa_test(test):
paragraphs = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
qa_test = do_qa_test(test)
 print('done')
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile)
 !pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False, 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.aug = albumentations.Compose([
albumentations.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], always_apply=True)
 ])
 def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(data_dir, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, OK = get_tiles(image, self.tile_mode)
 if self.rand:
idxes = np.random.choice(list(range(self.n_tiles)) , self.n_tiles, replace=False)
 else:
idxes = list(range(self.n_tiles))
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 img = images
img = np.transpose(img,(1, 2, 0))
img = 1 - img
img = cv2.resize(img,(768, 768))
write_image(f'{img_id}.png', img)
 img = skimage.io.MultiImage(f'{img_id}.png')[-1]
img = Image.fromarray(img ).convert("RGB")
 img = self.aug(image=np.array(img)) ["image"]
img = np.transpose(img,(2, 0, 1)).astype(np.float32)
 return { 'image': torch.tensor(img, dtype=torch.float)}<define_variables>
MODEL_PATH = '/kaggle/input/bert-pretrained-transformers/'
model = QuestionAnsweringModel('bert',
MODEL_PATH,
args={'reprocess_input_data': True,
'overwrite_output_dir': True,
'learning_rate': 3e-5,
'num_train_epochs': 2,
'max_seq_length': 128,
'doc_stride': 64,
'fp16': False,'train_batch_size': 64
},
use_cuda=use_cuda)
 model.train_model('data/train.json')
 predictions = model.predict(qa_test)
 predictions_df = pd.DataFrame.from_dict(predictions)
 sub_df['selected_text'] = predictions_df['answer']
print("File submitted successfully." )
<load_pretrained><EOS> 
sub_df['selected_text'][test_df['sentiment']=='neutral'] = test_df['text'][test_df['sentiment']=='neutral']
sub_df.to_csv('submission.csv', index=False )