Kaggle competition: IEEE's Signal Processing Society - Camera Model Identification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: ieees-signal-processing-society-camera-model-identification<define_variables>
%matplotlib inline
n = test_df['url'].apply(lambda x:('english.stackexchange.com' in x)).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.)<feature_engineering>
datagen = ImageDataGenerator(rescale=1./255)
 train_generator = datagen.flow_from_directory(
 '.. /input/train/',
batch_size=1,
class_mode='categorical' )
sub_df['question_type_spelling'] = spelling<save_to_csv>
x, y = train_generator.next()
plt.imshow(( x[0]*255 ).astype('uint8')) ;
print(list(train_generator.class_indices.keys())[np.argmax(y)] )
sub_df.to_csv("submission.csv", index=False )<install_modules>
X_data, Y_data = [], []
for _ in tqdm(range(2750)) :
x, y = train_generator.next()
X_data.append(x[0])
 Y_data.append(y[0])
 X_data = np.asarray(X_data)
 Y_data = np.asarray(Y_data )
!pip install.. /input/sacremoses/sacremoses-master/
!pip install.. /input/transformers/transformers-master/<define_variables>
def get_model() :
input_img = Input(( 256, 256, 3))
X = BatchNormalization()(input_img)
 X = Convolution2D(16,(3, 3), activation='relu' )(X)
 X = BatchNormalization()(X)
 X = Convolution2D(16,(3, 3), activation='relu' )(X)
 X = MaxPooling2D()(X)
 X = Convolution2D(32,(3, 3), activation='relu' )(X)
 X = BatchNormalization()(X)
 X = Convolution2D(32,(3, 3), activation='relu' )(X)
 X = GlobalMaxPooling2D()(X)
 X = BatchNormalization()(X)
 X = Dense(512, activation='relu' )(X)
 X = Dropout(0.2 )(X)
 X = Dense(10, activation='softmax' )(X)
 model = Model(inputs=input_img, outputs=X)
 model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['acc'])
 model.summary()
return model
DATA_DIR = '.. /input/google-quest-challenge'<load_from_csv>
model = get_model()
sub = pd.read_csv(f'{DATA_DIR}/sample_submission.csv')
 sub.head()<load_from_csv>
model_history = model.fit(X_data, Y_data, batch_size=10, epochs=3, validation_split=0.2,
callbacks=[EarlyStopping(monitor='val_acc', patience=3, verbose=1)] )
train = pd.read_csv(f'{DATA_DIR}/train.csv')
 train.head()<load_from_csv>
X_test = []
sub = pd.read_csv('.. /input/sample_submission.csv')
 for fname in tqdm(sub['fname']):
filepath = '.. /input/test/' + fname
X_test.append(img_to_array(load_img(filepath, target_size=(256, 256))))
X_test = np.asarray(X_test )
test = pd.read_csv(f'{DATA_DIR}/test.csv')
 test.head()<define_variables>
preds = model.predict(X_test, verbose=1)
 preds = np.argmax(preds, axis=1)
 preds = [list(train_generator.class_indices.keys())[p] for p in tqdm(preds)]
MAX_LEN = 512
SEP_TOKEN_ID = 102
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, df, model_type="bert-base-cased", max_len=512, content="Question_Answer", train_mode=True, labeled=True):
self.df = df
self.train_mode = train_mode
self.labeled = labeled
self.max_len = max_len
self.content = content
bert_tokenizer_path = '.. /input/pretrained-bert-models-for-pytorch/' + model_type + '-vocab.txt'
xlnet_tokenizer_path = '.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-spiece.model'
roberta_tokenizer_path = '.. /input/roberta-transformers-pytorch/roberta-base/vocab.json'
roberta_tokenizer_merges_file = '.. /input/roberta-transformers-pytorch/roberta-base/merges.txt'
if model_type == "bert-base-uncased":
self.tokenizer = BertTokenizer.from_pretrained(bert_tokenizer_path)
 elif model_type == "bert-base-cased":
self.tokenizer = BertTokenizer.from_pretrained(bert_tokenizer_path)
 elif model_type == "xlnet-base-cased":
self.tokenizer = XLNetTokenizer.from_pretrained(xlnet_tokenizer_path)
 elif model_type == "roberta-base":
self.tokenizer = RobertaTokenizer(vocab_file=roberta_tokenizer_path, merges_file=roberta_tokenizer_merges_file)
 def __getitem__(self, index):
row = self.df.iloc[index]
token_ids, seg_ids = self.get_token_ids(row)
 if self.labeled:
labels = self.get_label(row)
 return token_ids, seg_ids, labels
else:
return token_ids, seg_ids
def __len__(self):
return len(self.df)
 def select_tokens(self, tokens, max_num):
if len(tokens)<= max_num:
return tokens
if self.train_mode:
num_remove = len(tokens)- max_num
remove_start = random.randint(0, len(tokens)-num_remove-1)
 return tokens[:remove_start] + tokens[remove_start + num_remove:]
else:
return tokens[:max_num//2] + tokens[-(max_num - max_num//2):]
def trim_input_single_content(self, title, content, max_sequence_length=512,
t_max_len=30, c_max_len=512-30-4, num_token=3):
t = self.tokenizer.tokenize(title)
 c = self.tokenizer.tokenize(content)
 t_len = len(t)
 c_len = len(c)
 if(t_len+c_len+num_token)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
c_max_len = c_max_len + floor(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if c_max_len > c_len:
c_new_len = c_len
else:
c_new_len = c_max_len
if t_new_len+c_new_len+num_token > max_sequence_length:
raise ValueError("New sequence length should be less or equal than %d, but is %d"
%(max_sequence_length,(t_new_len+c_new_len+num_token)))
 if len(t)- t_new_len > 0:
t = t[:t_new_len//4] + t[len(t)-t_new_len+t_new_len//4:]
else:
t = t[:t_new_len]
if len(c)- c_new_len > 0:
c = c[:c_new_len//4] + c[len(c)-c_new_len+c_new_len//4:]
else:
c = c[:c_new_len]
if(len(t)+ len(c)+ num_token > max_sequence_length):
more_token = len(t)+ len(c)+ num_token - max_sequence_length
c = c[:(len(c)-more_token)]
return t, c
def trim_input(self, title, question, answer, max_sequence_length=MAX_LEN,
t_max_len=30, q_max_len=239, a_max_len=239, num_token=4):
t = self.tokenizer.tokenize(title)
 q = self.tokenizer.tokenize(question)
 a = self.tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+num_token)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+num_token > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+num_token)))
 if len(t)- t_new_len > 0:
t = t[:t_new_len//4] + t[len(t)-t_new_len+t_new_len//4:]
else:
t = t[:t_new_len]
if len(q)- q_new_len > 0:
q = q[:q_new_len//4] + q[len(q)-q_new_len+q_new_len//4:]
else:
q = q[:q_new_len]
if len(a)- a_new_len > 0:
a = a[:a_new_len//4] + a[len(a)-a_new_len+a_new_len//4:]
else:
a = a[:a_new_len]
return t, q, a
def get_token_ids(self, row):
num_token = 4
if self.content == "Question":
num_token -= 1
elif self.content == "Answer":
num_token -= 1
if self.content == "Question_Answer":
t_max_len=30
q_max_len=int(( self.max_len-t_max_len-num_token)/2)
 a_max_len=(self.max_len-t_max_len - num_token - int(( self.max_len-t_max_len-num_token)/2))
elif self.content == "Question":
t_max_len=30
q_max_len=self.max_len-t_max_len-num_token
a_max_len=0
elif self.content == "Answer":
t_max_len=30
q_max_len=0
a_max_len=self.max_len-t_max_len-num_token
else:
raise NotImplementedError
if self.content == "Question_Answer":
t_tokens, q_tokens, a_tokens = self.trim_input(row.question_title, row.question_body, row.answer, max_sequence_length=self.max_len, \
t_max_len=t_max_len, q_max_len=q_max_len, a_max_len=a_max_len, num_token=num_token)
 elif self.content == "Question":
t_tokens, c_tokens = self.trim_input_single_content(row.question_title, row.question_body, max_sequence_length=self.max_len, \
t_max_len=t_max_len, c_max_len=q_max_len, num_token=num_token)
 elif self.content == "Answer":
t_tokens, c_tokens = self.trim_input_single_content(row.question_title, row.answer, max_sequence_length=self.max_len, \
t_max_len=t_max_len, c_max_len=a_max_len, num_token=num_token)
 else:
raise NotImplementedError
if self.content == "Question_Answer":
tokens = ['[CLS]'] + t_tokens + ['[SEP]'] + q_tokens + ['[SEP]'] + a_tokens + ['[SEP]']
elif(( self.content == "Question")or(self.content == "Answer")) :
tokens = ['[CLS]'] + t_tokens + ['[SEP]'] + c_tokens + ['[SEP]']
else:
raise NotImplementedError
token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
 if len(token_ids)< self.max_len:
token_ids += [0] *(self.max_len - len(token_ids))
ids = torch.tensor(token_ids)
 seg_ids = self.get_seg_ids(ids)
 return ids, seg_ids
def get_seg_ids(self, ids):
seg_ids = torch.zeros_like(ids)
 seg_idx = 0
first_sep = True
for i, e in enumerate(ids):
seg_ids[i] = seg_idx
if e == self.tokenizer.sep_token_id:
if first_sep:
first_sep = False
else:
seg_idx = 1
pad_idx = torch.nonzero(ids == 0)
 seg_ids[pad_idx] = 0
return seg_ids
def get_label(self, row):
return torch.tensor(row[TARGET_COLUMNS].values.astype(np.float32))
def collate_fn(self, batch):
token_ids = torch.stack([x[0] for x in batch])
 seg_ids = torch.stack([x[1] for x in batch])
 if self.labeled:
labels = torch.stack([x[2] for x in batch])
 return token_ids, seg_ids, labels
else:
return token_ids, seg_ids
def get_test_loader(model_type="bert-base-cased", max_len=512, content="Question_Answer", batch_size=4):
df = pd.read_csv(f'{DATA_DIR}/test.csv')
 ds_test = QuestDataset(df, model_type, max_len=max_len, content=content, train_mode=False, labeled=False)
 loader = torch.utils.data.DataLoader(ds_test, batch_size=batch_size, shuffle=False, num_workers=2, collate_fn=ds_test.collate_fn, drop_last=False)
 loader.num = len(df)
 return loader, ds_test.tokenizer
def get_train_val_loaders(model_type="bert-base-cased", max_len=512, content="Question_Answer", batch_size=4, val_batch_size=4, ifold=0):
df = pd.read_csv(f'{DATA_DIR}/train.csv')
 df = shuffle(df, random_state=42)
 gkf = GroupKFold(n_splits=5 ).split(X=df.question_body, groups=df.question_body)
 for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold == ifold:
df_train = df.iloc[train_idx]
df_val = df.iloc[valid_idx]
break
print(df_train.shape)
 print(df_val.shape)
 ds_train = QuestDataset(df_train, model_type, max_len=max_len, content=content)
 train_loader = torch.utils.data.DataLoader(ds_train, batch_size=batch_size, shuffle=True, num_workers=2, collate_fn=ds_train.collate_fn, drop_last=True)
 train_loader.num = len(df_train)
 ds_val = QuestDataset(df_val, model_type, max_len=max_len, content=content, train_mode=False)
 val_loader = torch.utils.data.DataLoader(ds_val, batch_size=val_batch_size, shuffle=False, num_workers=2, collate_fn=ds_val.collate_fn, drop_last=False)
 val_loader.num = len(df_val)
 val_loader.df = df_val
return train_loader, val_loader, ds_train.tokenizer
def test_train_loader() :
loader, _, _ = get_train_val_loaders("xlnet-base-cased", 512, "Question", 4, 4, 1)
 for ids, seg_ids, labels in loader:
print(ids)
 print(seg_ids.numpy())
 print(labels)
 break
def test_test_loader() :
loader, _ = get_test_loader("roberta-base", 512, "Question", 4)
 for ids, seg_ids in loader:
print(ids)
 print(seg_ids)
 break<load_pretrained>
sub['camera'] = preds
sub.to_csv('sub.csv', index=False )