Kaggle competition: CareerCon 2019 - Help Navigate Robots
oov_embedding = test_model.get_weights() [0]
oov_embedding<merge>
%matplotlib inline
le = preprocessing.LabelEncoder()
warnings.filterwarnings('ignore')
 gc.enable()
def merge_embeddings(oov_embed, embed_vector):
for index, row in enumerate(embed_vector):
if row.sum() == 0:
embed_vector[index] = oov_embed[index]
return embed_vector
new_embedding = merge_embeddings(oov_embedding, embedding_vector )<train_model>
tr = pd.read_csv('.. /input/X_train.csv')
 te = pd.read_csv('.. /input/X_test.csv')
 target = pd.read_csv('.. /input/y_train.csv')
 ss = pd.read_csv('.. /input/sample_submission.csv')
 data = pd.read_csv('.. /input/X_train.csv' )
n_splits = 5
splits = list(KFold(n_splits=n_splits, shuffle=True, random_state=0 ).split(X_train,y_train))
predictions = np.zeros(shape=(X_test.shape[0], 1))
for n, fold in enumerate(splits):
print()
print("Fold {}".format(n + 1))
model_fold = "lstm_fold0{}.h5".format(n + 1)
 model_checkpoint = keras.callbacks.ModelCheckpoint(filepath=model_fold,
monitor="val_acc",
save_best_only=True,)
 callbacks_list = [model_checkpoint]
X_train_fold, y_train_fold = X_train[fold[0]], y_train[fold[0]]
X_val_fold, y_val_fold = X_train[fold[1]], y_train[fold[1]]
model = build_model(emb_init=new_embedding, emb_train=False)
 model.fit(X_train_fold, y_train_fold,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
validation_data=(X_val_fold, y_val_fold),
callbacks=callbacks_list,)
 model = keras.models.load_model(model_fold)
 predictions += model.predict(X_test)
 predictions /= n_splits<save_to_csv>
len(tr.measurement_number.value_counts() )
class_prediction = np.where(predictions > 0.5, 1, 0)
 submission = pd.DataFrame({"id": test_tweet["id"],
"target": class_prediction.flatten() })
 submission.to_csv("submission.csv", index=False )<import_modules>
print(len(tr.series_id.value_counts()))
print(128*3810,tr.shape )
RobertaConfig, RobertaModel, RobertaTokenizer)
<set_options>
len(te.measurement_number.value_counts() )
def set_seed(seed,n_gpu=0):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 if n_gpu > 0:
torch.cuda.manual_seed_all(seed)
 set_seed(0 )<set_options>
len(te.series_id.value_counts() )
class Config:
max_length=64
lr=1e-6
batch_size=64
train_epochs = 3
save_epoch = 3
max_grad_norm = 1
train_ratio=0.6
valid_ratio=0.3
device=torch.device("cuda")if torch.cuda.is_available() else torch.device("cpu")
 config = Config()<define_variables>
len(target.series_id.value_counts() )
def text_preprocess(raw_text,tokenizer,config):
MAX_LENGTH = config.max_length
url_pattern = re.compile(r'http://.*? |https://.*? ')
 at_pattern = re.compile(r'@.*? ')
 tag_pattern = re.compile(r'
face_pattern = re.compile(r'\x89.{2}')
 bracket_pattern = re.compile('\[.*?\]')
 repeat_pattern = re.compile(r'((.) \2+)')
 sep_pattern = re.compile(r'\[SEP\]')
 result = []
input_ids = []
attention_mask =[]
for text in tqdm(raw_text):
text += ' '
url_iter = url_pattern.finditer(text)
 for url_matcher in url_iter:
url = url_matcher.group().strip()
try:
title = None
except:
title = None
title = '[SEP] '+title+' [SEP]' if title else ''
text = text.replace(url,title)
 text = text.replace('
',' [SEP] ')
 text = at_pattern.sub('',text)
 text = face_pattern.sub('',text)
 tag_iter = tag_pattern.finditer(text)
 for tag_matcher in tag_iter:
tag = tag_matcher.group().strip()
text = text.replace(tag,'[SEP] '+tag[1:]+' [SEP]')
 text = text.replace('|',' [SEP] ')
 brackets = bracket_pattern.findall(text)
 for b in brackets:
if b != '[SEP]':
text = text.replace(b,'[SEP] '+b[1:-1]+' [SEP]')
 sep_iter = sep_pattern.finditer(text)
 new_text = ''
prev_idx = 0
for sep_matcher in sep_iter:
parts = text[prev_idx:sep_matcher.start() ]
parts = parts.strip()
new_text += parts+' [SEP] ' if parts else ''
prev_idx = sep_matcher.end()
text = new_text + text[prev_idx:]
rep_iter = repeat_pattern.finditer(text)
 new_text = ''
prev_idx = 0
for rep_matcher in rep_iter:
new_text += text[prev_idx:rep_matcher.start() +1]
prev_idx = rep_matcher.end()
text = new_text + text[prev_idx:]
text = text.replace(' ',' ')
 text = text.strip().strip('-')
 text = text.strip('-' ).strip()
while text[:5]=='[SEP]':
text = text[5:].strip()
while text[-5:]=='[SEP]':
text = text[:-5].strip()
text = text.replace('[SEP]',tokenizer.sep_token)
 text = text.lower()
result.append(text)
 iid,am = tokenizer(text ).values()
if len(iid)<=MAX_LENGTH:
pad_len = MAX_LENGTH-len(iid)
 iid += [1]*pad_len
am += [0]*pad_len
else:
iid = iid[:MAX_LENGTH]
am = am[:MAX_LENGTH]
input_ids.append(iid)
 attention_mask.append(am)
 return result,input_ids,attention_mask
def preprocess(df,tokenizer,config):
raw_text = df['text'].to_list()
sample_id = df['id'].to_list()
try:
target = df['target'].to_list()
except:
target = [0]*len(sample_id)
 result,input_ids,attention_mask = text_preprocess(raw_text,tokenizer,config)
 return {'id':sample_id,
'text':result,
'input_ids':input_ids,
'attention_mask':attention_mask,
'label':target}<load_from_csv>
target.surface.value_counts()
raw = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 raw.head()<load_pretrained>
def missing_data(data):
totalt = data.isnull().sum().sort_values(ascending=False)
 percent =(data.isnull().sum() /data.isnull().count() ).sort_values(ascending=False)
 return pd.concat([totalt, percent], axis=1, keys=['Total', 'Percent'] )
tokenizer = RobertaTokenizer.from_pretrained("roberta-base" )<string_transform>
target.groupby('group_id' ).surface.nunique().max()
raw = preprocess(raw,tokenizer,config )<randomize_order>
target['group_id'].nunique()
def kfold(data,k=5,shuffle=True,file_prefix='/kaggle/working/kfold',file_suffix='tuple2'):
keys = list(data.keys())
 total_idx = list(range(len(data[keys[0]])))
 total = len(total_idx)
 valid_num = int(total/k)
 if shuffle:
random.shuffle(total_idx)
 for i in trange(k):
train_data = {}
valid_data = {}
for key in data.keys() :
train_data[key] = []
valid_data[key] = []
valid_idx = total_idx[valid_num*(k-1):valid_num*k]
for idx in valid_idx:
for key in valid_data.keys() :
valid_data[key].append(data[key][idx])
 train_idx = total_idx[:valid_num*(k-1)]+total_idx[valid_num*k:]
for idx in train_idx:
for key in train_data.keys() :
train_data[key].append(data[key][idx])
 with open('{pf}_{num}.{sf}'.format(pf=file_prefix,num=i,sf=file_suffix),'wb')as f:
pickle.dump(( train_data,valid_data),f)
 def dataset_split(data,train_ratio=0.6,valid_ratio=0.3,shuffle=True):
assert train_ratio+valid_ratio<=1 , "The train_ratio + valid_ratio should not be greater than 1"
keys = list(data.keys())
 total_idx = list(range(len(data[keys[0]])))
 total = len(total_idx)
 train_num = int(total*train_ratio)
 valid_num = int(total*valid_ratio)
 if shuffle:
random.shuffle(total_idx)
 train_data = {}
valid_data = {}
test_data = {}
for k in data.keys() :
train_data[k] = []
valid_data[k] = []
test_data[k] = []
for idx in total_idx[:train_num]:
for k in train_data.keys() :
train_data[k].append(data[k][idx])
 for idx in total_idx[train_num:train_num+valid_num]:
for k in valid_data.keys() :
valid_data[k].append(data[k][idx])
 for idx in total_idx[train_num+valid_num:]:
for k in test_data.keys() :
test_data[k].append(data[k][idx])
 return train_data,valid_data,test_data<feature_engineering>
group27 = target[ target.group_id == 27]
print(group27 )
class MyDataset(Dataset):
def __init__(self, train_data):
self.id = train_data['id']
self.text = train_data['text']
self.input_ids = train_data['input_ids']
self.attention_mask = train_data['attention_mask']
self.label = train_data['label']
assert len(self.text)==len(self.label)
 self.idx = list(range(len(self.label)))
 def __getitem__(self, index):
idx = index
return {
'id':torch.tensor(self.id[idx],dtype=torch.long),
'text':self.text[idx],
'input_ids':torch.tensor(self.input_ids[idx],dtype=torch.long),
'attention_mask':torch.tensor(self.attention_mask[idx],dtype=torch.long),
'label':torch.tensor(self.label[idx],dtype=torch.long)
 }
def __len__(self):
return len(self.label )<choose_model_class>
target.group_id.value_counts()
class ClassificationHead(nn.Module):

 def __init__(self, input_size, output_size, d_model_0, d_model_1, dropout_prob):
super().__init__()
self.norm= nn.BatchNorm1d(input_size)
 self.dense = nn.Linear(input_size, d_model_0)
 self.norm_1= nn.BatchNorm1d(d_model_0)
 self.dropout = nn.Dropout(dropout_prob)
 self.dense_1 = nn.Linear(d_model_0, d_model_1)
 self.norm_2= nn.BatchNorm1d(d_model_1)
 self.out_proj = nn.Linear(d_model_1, output_size)
 def forward(self, features, **kwargs):
x = self.norm(features)
 x = self.dropout(x)
 x = self.dense(x)
 x = torch.relu(self.norm_1(x))
x = self.dropout(x)
 x = self.dense_1(x)
 x = torch.relu(self.norm_2(x))
x = self.dropout(x)
 x = self.out_proj(x)
 return x<load_pretrained>
train_group = pd.merge(tr,target, on = 'series_id', how = 'left')
 train_group[:5]
class Model0(nn.Module):
def __init__(self,encoder,config):
super().__init__()
self.encoder = encoder
self.classifier=ClassificationHead(768,2,256,64,0.2)
 self.classifier.apply(self._init_weights)
 def _init_weights(self, module):

 if isinstance(module,(nn.Linear, nn.Embedding)) :
module.weight.data.normal_(mean=0.0, std=0.02)
 def forward(self,input_ids,attention_mask,label=None):
hidden_states = self.encoder(input_ids=input_ids,attention_mask=attention_mask)[1]
logits = self.classifier(hidden_states)
 loss_fct = CrossEntropyLoss()
if label is None:
loss = 0
else:
loss = loss_fct(logits, label)
 prob=torch.softmax(logits,-1)
 return prob,loss<choose_model_class>
del serie1
gc.collect()
class Model1(nn.Module):
def __init__(self,encoder,config):
super().__init__()
self.encoder = encoder
self.conv1d = nn.Conv1d(768*2,1024,config.max_length)
 self.classifier=ClassificationHead(1024,2,512,256,0.2)
 self.classifier.apply(self._init_weights)
 def _init_weights(self, module):

 if isinstance(module,(nn.Linear, nn.Embedding)) :
module.weight.data.normal_(mean=0.0, std=0.02)
 def forward(self,input_ids,attention_mask,label=None):
hidden_states = self.encoder(input_ids=input_ids,attention_mask=attention_mask)[-1][-2:]
hidden_states = torch.cat(hidden_states,-1)
 hidden_states = hidden_states.permute(0,2,1)
 hidden_states = self.conv1d(hidden_states)
 hidden_states = torch.squeeze(hidden_states)
 logits = self.classifier(hidden_states)
 loss_fct = CrossEntropyLoss()
if label is None:
loss = 0
else:
loss = loss_fct(logits, label)
 prob=torch.softmax(logits,-1)
 return prob,loss<categorify>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
class Model2(nn.Module):
def __init__(self,encoder,config):
super().__init__()
self.encoder = encoder
self.conv1d = nn.Conv1d(768,1024,config.max_length)
 self.classifier=ClassificationHead(1024,2,512,256,0.2)
 self.classifier.apply(self._init_weights)
 def _init_weights(self, module):

 if isinstance(module,(nn.Linear, nn.Embedding)) :
module.weight.data.normal_(mean=0.0, std=0.02)
 def forward(self,input_ids,attention_mask,label=None):
hidden_states = self.encoder(input_ids=input_ids,attention_mask=attention_mask)[-1][-2:]
hidden_states = hidden_states[0] + hidden_states[1]
hidden_states = hidden_states.permute(0,2,1)
 hidden_states = self.conv1d(hidden_states)
 hidden_states = torch.squeeze(hidden_states)
 logits = self.classifier(hidden_states)
 loss_fct = CrossEntropyLoss()
if label is None:
loss = 0
else:
loss = loss_fct(logits, label)
 prob=torch.softmax(logits,-1)
 return prob,loss<find_best_model_class>
def fe_step0(actual):
actual['norm_quat'] =(actual['orientation_X']**2 + actual['orientation_Y']**2 + actual['orientation_Z']**2 + actual['orientation_W']**2)
 actual['mod_quat'] =(actual['norm_quat'])**0.5
actual['norm_X'] = actual['orientation_X'] / actual['mod_quat']
actual['norm_Y'] = actual['orientation_Y'] / actual['mod_quat']
actual['norm_Z'] = actual['orientation_Z'] / actual['mod_quat']
actual['norm_W'] = actual['orientation_W'] / actual['mod_quat']
return actual
def infer(model,test_data,tokenizer,config,save=False,save_path='submission.csv'):
model.eval()
test_dl = DataLoader(MyDataset(test_data),shuffle=False,batch_size=config.batch_size)
 preds = []
idx = []
dl_iter = tqdm(test_dl,desc='[Infering]',position=0)
 device = config.device
for data in dl_iter:
text = data['text']
input_ids = data['input_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 model.eval()
prob,loss = model(input_ids,attention_mask)
 pred = torch.argmax(prob,-1 ).detach().cpu().numpy()
preds.append(pred)
 idx.append(data['id'].detach().cpu().numpy())
 dl_iter.set_postfix(id=data['id'][0].item())
 preds = np.concatenate(preds)
 idx = np.concatenate(idx)
 if save:
pd.DataFrame({'id':idx,'preds':preds} ).to_csv(save_path)
 return idx,preds<load_pretrained>
def fe_step1(actual):

 x, y, z, w = actual['norm_X'].tolist() , actual['norm_Y'].tolist() , actual['norm_Z'].tolist() , actual['norm_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 actual['euler_x'] = nx
actual['euler_y'] = ny
actual['euler_z'] = nz
return actual
def train(model,config,save_name='model',fold_num=0):
with open('kfold_{fold_num}.tuple2'.format(fold_num=fold_num),'rb')as f:
train_data,valid_data = pickle.load(f)
 train_dl = DataLoader(MyDataset(train_data),shuffle=True,batch_size=config.batch_size)
 valid_dl = DataLoader(MyDataset(valid_data),shuffle=True,batch_size=config.batch_size)
 optimizer = AdamW(model.parameters() , lr=1e-5, eps=1e-8,weight_decay=0.08)
 scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=int(len(train_dl)*config.train_epochs*0.2),num_training_steps=len(train_dl)*config.train_epochs)
 step_count = 0
total_loss = 0
device = torch.device("cuda")if torch.cuda.is_available() else torch.device("cpu")
 model.to(device)
 model.train()
total_f1_score = 0
for epoch in range(config.train_epochs):
dl_iter = tqdm(train_dl,desc='[Training]',position=0)
 for data in dl_iter:
text = data['text']
input_ids = data['input_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 label = data['label'].to(device)
 model.train()
prob,loss = model(input_ids,attention_mask,label)
 loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , config.max_grad_norm)
 optimizer.step()
optimizer.zero_grad()
scheduler.step()
step_count += 1
total_loss += loss.item()
avg_loss = total_loss/step_count
pred = torch.argmax(prob,-1 ).detach().cpu().numpy()
label = label.detach().cpu().numpy()
total_f1_score += sklearn.metrics.f1_score(label, pred)
 f1_score = total_f1_score/step_count
dl_iter.set_postfix(epoch=epoch,loss=loss.item() ,avg_loss=avg_loss,f1_score=f1_score)
 model.eval()
f1_score = eval(model,valid_dl,device,-1)
 tqdm.write('[epcoh_{epoch}] f1_score: {f1_score}'.format(epoch=epoch,f1_score=f1_score))
if(epoch+1)%config.save_epoch == 0:
torch.save(model,'{model_name}_fold{fold_num}_epoch{epoch}.pt'.format(model_name=save_name,fold_num=fold_num,epoch=epoch))
model.train()<load_pretrained>
def feat_eng(data):
df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 + data['angular_velocity_Z']**2)** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 + data['linear_acceleration_Z']**2)**0.5
data['totl_xyz'] =(data['orientation_X']**2 + data['orientation_Y']**2 + data['orientation_Z']**2)**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
def mean_change_of_abs_change(x):
return np.mean(np.diff(np.abs(np.diff(x))))
for col in data.columns:
if col in ['row_id','series_id','measurement_number']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_mean_change_of_abs_change'] = data.groupby('series_id')[col].apply(mean_change_of_abs_change)
 df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df
RobertaModel.from_pretrained('roberta-base' )<load_pretrained>
data.fillna(0,inplace=True)
 test.fillna(0,inplace=True)
 data.replace(-np.inf,0,inplace=True)
 data.replace(np.inf,0,inplace=True)
 test.replace(-np.inf,0,inplace=True)
 test.replace(np.inf,0,inplace=True )
try:
os.mkdir('/kaggle/working/models/')
 except:
pass
config.train_epochs = 3
config.save_epoch = 3
for fn in range(5):
tqdm.write('='*10+'FOLD_{fn}'.format(fn=fn)+'='*10)
 encoder = RobertaModel.from_pretrained('roberta-base')
 encoder.config.output_hidden_states = True
model = Model2(encoder,config)
 train(model,config,save_name='/kaggle/working/models/model2',fold_num=fn)
 del model
del encoder<load_from_csv>
target['surface'] = le.fit_transform(target['surface'] )
test_raw = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 test_data = preprocess(test_raw,tokenizer,config)
 vote = []
model_num = 0
for home, dirs, files in os.walk('/kaggle/working/models/'):
for filename in files:
if filename.endswith('.pt'):
model_num += 1
model = torch.load(os.path.join(home,filename))
idx,preds = infer(model,test_data,tokenizer,config,save_path='{mn}.csv'.format(mn=filename[:-3]))
vote.append(preds)
 del model
vote = np.array(vote ).T
result = np.sum(vote,axis=1)[:,None]
threshold = int(model_num/2)
 print('Model Num: {mn}
Threshold: {ts}'.format(mn=model_num,ts=threshold))
result = np.apply_along_axis(lambda x:1 if x>threshold else 0,1,result)
 pd.DataFrame({'id':idx,'target':result} ).to_csv('/kaggle/working/submission.csv',index=0 )<import_modules>
target['surface'].value_counts()
stop = set(stopwords.words('english'))
print('Imports Done' )<load_from_csv>
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=20)
 predicted = np.zeros(( test.shape[0],9))
measured= np.zeros(( data.shape[0]))
score = 0
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 print('Train df shape: ' + str(train.shape))
test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 print('Validation/Test df shape: ' + str(test.shape))
test.head()<categorify>
for times,(trn_idx, val_idx)in enumerate(folds.split(data.values,target['surface'].values)) :
model = RandomForestClassifier(n_estimators=500)
 model.fit(data.iloc[trn_idx],target['surface'][trn_idx])
 measured[val_idx] = model.predict(data.iloc[val_idx])
 predicted += model.predict_proba(test)/folds.n_splits
score += model.score(data.iloc[val_idx],target['surface'][val_idx])
 print("Fold: {} score: {}".format(times,model.score(data.iloc[val_idx],target['surface'][val_idx])))
 importances = model.feature_importances_
indices = np.argsort(importances)
 features = data.columns
if model.score(data.iloc[val_idx],target['surface'][val_idx])> 0.91000:
hm = 30
plt.figure(figsize=(7, 10))
plt.title('Feature Importances')
 plt.barh(range(len(indices[:hm])) , importances[indices][:hm], color='b', align='center')
 plt.yticks(range(len(indices[:hm])) , [features[i] for i in indices])
 plt.xlabel('Relative Importance')
 plt.show()
gc.collect()
df = pd.concat([train, test])
 df.shape
def remove_url(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 def remove_punctuation(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 df['text']=df['text'].apply(lambda x : remove_punctuation(x))
df['text']=df['text'].apply(lambda x : remove_emoji(x))
df['text']=df['text'].apply(lambda x : remove_html(x))
df['text']=df['text'].apply(lambda x : remove_url(x))
df.head()<randomize_order>
print('Avg Accuracy', score / folds.n_splits )
def create_glove_corpus() :
corpus=[]
for tweet in tqdm(df['text']):
words=[word.lower() for word in word_tokenize(tweet)]
corpus.append(words)
 return corpus
corpus = create_glove_corpus()
embedding_dict={}
with open('.. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt','r')as f:
for line in f:
values=line.split()
word = values[0]
vectors=np.asarray(values[1:],'float32')
 embedding_dict[word]=vectors
f.close()
MAX_LEN=50
tokenizer_obj=Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences=tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad=pad_sequences(sequences,maxlen=MAX_LEN,truncating='post',padding='post')
 word_index=tokenizer_obj.word_index
print('Number of unique words:',len(word_index))
num_words=len(word_index)+1
embedding_matrix=np.zeros(( num_words,100))
for word,i in tqdm(word_index.items()):
if i < num_words:
emb_vec=embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i]=emb_vec
tweet_pad[0][0:]<split>
confusion_matrix(measured,target['surface'] )
model=Sequential()
embedding=Embedding(num_words,100,embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN,trainable=False)
 model.add(embedding)
 model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
optimzer=Adam(learning_rate=3e-4)
 model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'])
 model.summary()
glove_train=tweet_pad[:train.shape[0]]
glove_test=tweet_pad[train.shape[0]:]
x_train,x_test,y_train,y_test=train_test_split(glove_train,train['target'].values,test_size=0.2)
 print('Shape of train',x_train.shape)
 print("Shape of Validation ",x_test.shape )<train_model>
ss['surface'] = le.inverse_transform(predicted.argmax(axis=1))
ss.to_csv('submission.csv', index=False)
 ss.head()
<save_to_csv><EOS>