Kaggle competition: Natural Language Processing with Disaster Tweets
params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'max_depth': 16,
'num_leaves': 31,
'learning_rate': 0.025,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'verbose': 0,
'num_threads': 1,
'lambda_l2': 1,
'min_gain_to_split': 0,
}
model2 = lgb.train(
 params,
lgb.Dataset(training_features, training_targets),
num_boost_round=10000,
valid_sets=[lgb.Dataset(validation_features, validation_targets)],
verbose_eval=100,
early_stopping_rounds=100
)<save_to_csv>
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )
submission = pd.DataFrame()
submission['id'] = test['id']
submission['project_is_approved'] = model2.predict(test_features, num_iteration=model2.best_iteration)
 submission.to_csv('submission.csv', index=False )<feature_engineering>
df_train=pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 df_test=pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )
target = 'project_is_approved'
Ttr['tr'] = 1; Tts['tr'] = 0
Ttr['ts'] = 0; Tts['ts'] = 1
T = pd.concat(( Ttr,Tts))
T.loc[T.project_essay_4.isnull() , ['project_essay_4','project_essay_2']] = \
T.loc[T.project_essay_4.isnull() , ['project_essay_2','project_essay_4']].values
T[['project_essay_2','project_essay_3']] = T[['project_essay_2','project_essay_3']].fillna('')
 T['project_essay_1'] = T.apply(lambda row: ' '.join([str(row['project_essay_1']),
str(row['project_essay_2'])]), axis=1)
 T['project_essay_2'] = T.apply(lambda row: ' '.join([str(row['project_essay_3']),
str(row['project_essay_4'])]), axis=1)
 T = T.drop(['project_essay_3', 'project_essay_4'], axis=1)
<merge>
def preprocess(text):
text=text.lower()
text = re.sub(r'https?:\/\/.*[\r
]*', '', text)
 text = re.sub(r'http?:\/\/.*[\r
]*', '', text)
 text=text.replace(r'&amp;?',r'and')
 text=text.replace(r'&lt;',r'<')
 text=text.replace(r'&gt;',r'>')
 text = re.sub(r"(?:\@)\w+", '', text)
 text=text.encode("ascii",errors="ignore" ).decode()
text=re.sub(r'[:"
text=re.sub(r'[!]+','!',text)
 text=re.sub(r'[?]+','?',text)
 text=re.sub(r'[.]+','.',text)
 text=re.sub(r"'","",text)
 text=re.sub(r"\(","",text)
 text=re.sub(r"\)","",text)
 text=" ".join(text.split())
 return text
df_train['text'] = df_train['text'].apply(preprocess)
 df_test['text'] = df_test['text'].apply(preprocess)
 df_train=df_train[df_train["text"]!='']
R['priceAll'] = R['quantity']*R['price']
newR = R.groupby('id' ).agg({'description':'count',
'quantity':'sum',
'price':'sum',
'priceAll':'sum'} ).rename(columns={'description':'items'})
 newR['avgPrice'] = newR.priceAll / newR.quantity
numFeatures = ['items', 'quantity', 'price', 'priceAll', 'avgPrice']
for func in ['min', 'max', 'mean']:
newR = newR.join(R.groupby('id' ).agg({'quantity':func,
'price':func,
'priceAll':func} ).rename(
 columns={'quantity':func+'Quantity',
'price':func+'Price',
'priceAll':func+'PriceAll'} ).fillna(0))
numFeatures += [func+'Quantity', func+'Price', func+'PriceAll']
newR = newR.join(R.groupby('id' ).agg(
 {'description':lambda x:' '.join(x.values.astype(str)) } ).rename(
 columns={'description':'resource_description'}))
T = T.join(newR, on='id')
 T['price_category'] = pl.digitize(T.priceAll, [0, 50, 100, 250, 500, 1000, pl.inf])
 numFeatures.append('price_category')
 for c in ['Quantity', 'Price', 'PriceAll']:
T['max%s_min%s'%(c,c)] = T['max%s'%c] - T['min%s'%c]
numFeatures.append('max%s_min%s'%(c,c))
del Ttr, Tts, R, newR
gc.collect() ;<categorify>
df_train["target"].value_counts()
le = LabelEncoder()
T['teacher_id'] = le.fit_transform(T['teacher_id'])
 T['teacher_gender_unknown'] = T.teacher_prefix.apply(lambda x:int(x not in ['Ms.', 'Mrs.', 'Mr.']))
numFeatures += ['teacher_number_of_previously_posted_projects','teacher_id','teacher_gender_unknown']
statFeatures = []
for col in ['school_state', 'teacher_id', 'teacher_prefix', 'teacher_gender_unknown', 'project_grade_category', 'project_subject_categories', 'project_subject_subcategories', 'teacher_number_of_previously_posted_projects']:
Stat = T[['id', col]].groupby(col ).agg('count' ).rename(columns={'id':col+'_stat'})
 Stat /= Stat.sum()
T = T.join(Stat, on=col)
 statFeatures.append(col+'_stat' )<feature_engineering>
texts = df_train.text.values
labels = df_train.target.values
%%time
textColumns = ['project_essay_1', 'project_essay_2', 'project_resource_summary', 'resource_description', 'project_title']
def getSentFeat(s):
sent = TextBlob(s ).sentiment
return(sent.polarity, sent.subjectivity)
 print('sentimental analysis')
 with Pool(4)as p:
for col in textColumns:
temp = pl.array(list(p.map(getSentFeat, T[col])))
 T[col+'_pol'] = temp[:,0]
T[col+'_sub'] = temp[:,1]
numFeatures += [col+'_pol', col+'_sub']
print('key words')
 KeyChars = ['!', '\?', '@', '
'\.', ':', ';', ',', '/', '\\\\r', '\\\\t', '\"', '\.\.\.', 'etc', 'http', 'poor',
'military', 'traditional', 'charter', 'head start', 'magnet', 'year-round', 'alternative',
'art', 'book', 'basics', 'computer', 'laptop', 'tablet', 'kit', 'game', 'seat',
'food', 'cloth', 'hygiene', 'instraction', 'technolog', 'lab', 'equipment',
'music', 'instrument', 'nook', 'desk', 'storage', 'sport', 'exercise', 'trip', 'visitor',
'my students', 'our students', 'my class', 'our class']
for col in textColumns:
for c in KeyChars:
T[col+'_'+c] = T[col].apply(lambda x: len(re.findall(c, x.lower())))
 numFeatures.append(col+'_'+c)
 print('num words')
 for col in textColumns:
T['n_'+col] = T[col].apply(lambda x: len(x.split()))
numFeatures.append('n_'+col)
 T['nUpper_'+col] = T[col].apply(lambda x: sum([s.isupper() for s in list(x)]))
numFeatures.append('nUpper_'+col)
 print('word tags')
 Tags = ['CC', 'CD', 'DT', 'IN', 'JJ', 'LS', 'MD', 'NN', 'NNS', 'NNP', 'NNPS',
'PDT', 'POS', 'PRP', 'PRP$', 'RB', 'RBR', 'RBS', 'RP', 'SYM', 'TO',
'UH', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', 'WDT', 'WP', 'WP$', 'WRB']
def getTagFeat(s):
d = Counter([t[1] for t in pos_tag(s.split())])
 return [d[t] for t in Tags]
with Pool(4)as p:
for col in textColumns:
temp = pl.array(list(p.map(getTagFeat, T[col])))
 for i, t in enumerate(Tags):
if temp[:,i].sum() == 0:
continue
T[col+'_'+t] = temp[:, i]
numFeatures += [col+'_'+t]
print('common words')
 for i, col1 in enumerate(textColumns[:-1]):
for col2 in textColumns[i+1:]:
T['%s_%s_common' %(col1, col2)] = T.apply(lambda row:len(set(re.split('\W', row[col1].lower())).intersection(re.split('\W', row[col2].lower()))), axis=1)
 numFeatures.append('%s_%s_common' %(col1, col2)) 
<feature_engineering>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
 model = BertForSequenceClassification.from_pretrained('bert-base-uncased',num_labels=2)
 model.cuda()
dateCol = 'project_submitted_datetime'
def getTimeFeatures(T):
T['year'] = T[dateCol].apply(lambda x: x.year)
 T['month'] = T[dateCol].apply(lambda x: x.month)
 T['day'] = T[dateCol].apply(lambda x: x.day)
 T['dow'] = T[dateCol].apply(lambda x: x.dayofweek)
 T['hour'] = T[dateCol].apply(lambda x: x.hour)
 T['days'] =(T[dateCol]-T[dateCol].min() ).apply(lambda x: x.days)
 return T
T[dateCol] = pd.to_datetime(T[dateCol])
 T = getTimeFeatures(T)
 P_tar = T[T.tr==1][target].mean()
timeFeatures = ['year', 'month', 'day', 'dow', 'hour', 'days']
for col in timeFeatures:
Stat = T[['id', col]].groupby(col ).agg('count' ).rename(columns={'id':col+'_stat'})
 Stat /= Stat.sum()
T = T.join(Stat, on=col)
 statFeatures.append(col+'_stat')
 numFeatures += timeFeatures
numFeatures += statFeatures<feature_engineering>
indices=tokenizer.batch_encode_plus(texts,max_length=64,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids=indices["input_ids"]
attention_masks=indices["attention_mask"]
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=42, test_size=0.2)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,
random_state=42, test_size=0.2 )
%%time
T2 = T[numFeatures+['id','tr','ts',target]].copy()
Ttr = T2[T.tr==1]
Tar_tr = Ttr[target].values
n = 10
inx = [pl.randint(0, Ttr.shape[0], int(Ttr.shape[0]/n)) for k in range(n)]
Corr = {}
for c in numFeatures:
C1,P1=pl.nanmean([pearsonr(Tar_tr[inx[k]],(1+Ttr[c].iloc[inx[k]])) for k in range(n)], 0)
 C2,P2=pl.nanmean([pearsonr(Tar_tr[inx[k]], 1/(1+Ttr[c].iloc[inx[k]])) for k in range(n)], 0)
 if P2<P1:
T2[c] = 1/(1+T2[c])
 Corr[c] = [C2,P2]
else:
T2[c] = 1+T2[c]
Corr[c] = [C1,P1]
polyCol = []
thrP = 0.01
thrC = 0.02
print('columns \t\t\t Corr1 \t\t Corr2 \t\t Corr Combined')
 for i, c1 in enumerate(numFeatures[:-1]):
C1, P1 = Corr[c1]
for c2 in numFeatures[i+1:]:
C2, P2 = Corr[c2]
V = T2[c1] * T2[c2]
Vtr = V[T2.tr==1].values
C, P = pl.nanmean([pearsonr(Tar_tr[inx[k]], Vtr[inx[k]])for k in range(n)], 0)
 if P<thrP and abs(C)- max(abs(C1),abs(C2)) > thrC:
T[c1+'_'+c2+'_poly'] = V
polyCol.append(c1+'_'+c2+'_poly')
 print(c1+'_'+c2, '\t\t(%g, %g)\t(%g, %g)\t(%g, %g)'%(C1,P1, C2,P2, C,P))
numFeatures += polyCol
print(len(numFeatures))
del T2, Ttr
gc.collect() ;<feature_engineering>
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels, dtype=torch.long)
 validation_labels = torch.tensor(validation_labels, dtype=torch.long)
 train_masks = torch.tensor(train_masks, dtype=torch.long)
 validation_masks = torch.tensor(validation_masks, dtype=torch.long )
def getCatFeatures(T, Col):
vectorizer = CountVectorizer(binary=True,
ngram_range=(1,1),
tokenizer=lambda x:[a.strip() for a in x.split(',')])
 return vectorizer.fit_transform(T[Col].fillna(''))
X_tp = getCatFeatures(T, 'teacher_prefix')
 X_ss = getCatFeatures(T, 'school_state')
 X_pgc = getCatFeatures(T, 'project_grade_category')
 X_psc = getCatFeatures(T, 'project_subject_categories')
 X_pssc = getCatFeatures(T, 'project_subject_subcategories')
 X_cat = hstack(( X_tp, X_ss, X_pgc, X_psc, X_pssc))
del X_tp, X_ss, X_pgc, X_psc, X_pssc<string_transform>
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size )
%%time
p = PorterStemmer()
def wordPreProcess(sentence):
return ' '.join([p.stem(x.lower())for x in re.split('\W', sentence)if len(x)>= 1])
 def getTextFeatures(T, Col, max_features=10000, ngrams=(1,2), verbose=True):
if verbose:
print('processing: ', Col)
 vectorizer = CountVectorizer(stop_words=None,
preprocessor=wordPreProcess,
max_features=max_features,
binary=True,
ngram_range=ngrams)
 X = vectorizer.fit_transform(T[Col])
 return X, vectorizer.get_feature_names()
n_es1, n_es2, n_prs, n_rd, n_pt = 3000, 8000, 2000, 3000, 1000
X_es1, feat_es1 = getTextFeatures(T, 'project_essay_1', max_features=n_es1)
 X_es2, feat_es2 = getTextFeatures(T, 'project_essay_2', max_features=n_es2)
 X_prs, feat_prs = getTextFeatures(T, 'project_resource_summary', max_features=n_prs)
 X_rd, feat_rd = getTextFeatures(T, 'resource_description', max_features=n_rd, ngrams=(1,3))
X_pt, feat_pt = getTextFeatures(T, 'project_title', max_features=n_pt)
 X_txt = hstack(( X_es1, X_es2, X_prs, X_rd, X_pt))
del X_es1, X_es2, X_prs, X_rd, X_pt 
<prepare_x_and_y>
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat)
 def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))
X = hstack(( X_txt, X_cat, StandardScaler().fit_transform(T[numFeatures].fillna(0)))).tocsr()
Xtr = X[pl.find(T.tr==1), :]
Xts = X[pl.find(T.ts==1), :]
Ttr_tar = T[T.tr==1][target].values
Tts = T[T.ts==1][['id',target]]
Yts = []
del T
del X
gc.collect() ; 
<choose_model_class>
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(train_dataloader)
 loss_values.append(avg_train_loss)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epoch took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )
def breakInput(X1):
X2 = []
i = 0
for n in [n_es1, n_es2, n_prs, n_rd, n_pt, X_cat.shape[1], len(numFeatures)]:
X2.append(X1[:,i:i+n])
 i += n
return X2
def getModel(HLs, Drop=0.25, OP=optimizers.Adam()):
temp = []
inputs_txt = []
for n in [n_es1, n_es2, n_prs, n_rd, n_pt]:
input_txt = Input(( n,))
X_feat = Dropout(Drop )(input_txt)
 X_feat = Dense(int(n/100), activation="linear" )(X_feat)
 X_feat = Dropout(Drop )(X_feat)
 temp.append(X_feat)
 inputs_txt.append(input_txt)
 x_1 = concatenate(temp)
 x_1 = Dense(50, activation="relu" )(x_1)
 x_1 = Dropout(Drop )(x_1)
 input_cat = Input(( X_cat.shape[1],))
x_2 = Embedding(2, 10, input_length=X_cat.shape[1] )(input_cat)
 x_2 = SpatialDropout1D(Drop )(x_2)
 x_2 = Flatten()(x_2)
 input_num = Input(( len(numFeatures),))
x_3 = Dropout(Drop )(input_num)
 x = concatenate([x_1, x_2, x_3])
 for HL in HLs:
x = Dense(HL, activation="relu" )(x)
 x = Dropout(Drop )(x)
 output = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inputs_txt+[input_cat, input_num], outputs=output)
 model.compile(
 optimizer=OP,
loss='binary_crossentropy',
metrics=['binary_accuracy'])
 return model
def trainNN(X_train, X_val, Tar_train, Tar_val, HL=[50], Drop=0.5, OP=optimizers.Adam()):
file_path='NN.h5'
checkpoint = ModelCheckpoint(file_path, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True, mode='min')
 early = EarlyStopping(monitor="val_loss", mode="min", patience=6)
 lr_reduced = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=2,
verbose=1,
epsilon=3e-4,
mode='min')
 model = getModel(HL, Drop, OP)
 model.fit(breakInput(X_train), Tar_train, validation_data=(breakInput(X_val), Tar_val),
verbose=2, epochs=50, batch_size=1000, callbacks=[early, lr_reduced, checkpoint])
 model.load_weights(file_path)
 return model
params_xgb = {
'eta': 0.05,
'max_depth': 4,
'subsample': 0.85,
'colsample_bytree': 0.25,
'min_child_weight': 3,
'objective': 'binary:logistic',
'eval_metric': 'auc',
'seed': 0,
'silent': 1,
}
params_lgb = {
'boosting_type': 'dart',
'objective': 'binary',
'metric': 'auc',
'max_depth': 10,
'learning_rate': 0.05,
'feature_fraction': 0.25,
'bagging_fraction': 0.85,
'seed': 0,
'verbose': 0,
}
nCV = 1
for i in range(21, 22):
gc.collect()
X_train, X_val, Tar_train, Tar_val = train_test_split(Xtr, Ttr_tar, test_size=0.15, random_state=i, stratify=Ttr_tar)
 dtrain = xgb.DMatrix(X_train, label=Tar_train)
 dval = xgb.DMatrix(X_val, label=Tar_val)
 watchlist = [(dtrain, 'train'),(dval, 'valid')]
model = xgb.train(params_xgb, dtrain, 5000, watchlist, maximize=True, verbose_eval=200, early_stopping_rounds=200)
 Yvl1 = model.predict(dval)
 Yts1 = model.predict(xgb.DMatrix(Xts))
dtrain = lgb.Dataset(X_train, Tar_train)
 dval = lgb.Dataset(X_val, Tar_val)
 model = lgb.train(params_lgb, dtrain, num_boost_round=10000, valid_sets=[dtrain, dval], early_stopping_rounds=200, verbose_eval=200)
 Yvl2 = model.predict(X_val)
 Yts2 = model.predict(Xts)
 model = trainNN(X_train, X_val, Tar_train, Tar_val, HL=[50], Drop=0.5, OP=optimizers.Adam())
 Yvl3 = model.predict(breakInput(X_val)).squeeze()
Yts3 = model.predict(breakInput(Xts)).squeeze()
M = LinearRegression()
M.fit(pl.array([Yvl1, Yvl2, Yvl3] ).T, Tar_val)
 Yts.append(M.predict(pl.array([Yts1, Yts2, Yts3] ).T)) 
<save_to_csv>
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
preds=[]
true=[]
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
preds.append(logits)
 true.append(label_ids)
 tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print(" Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
print(" Validation took: {:}".format(format_time(time.time() - t0)))
Tts[target] = MinMaxScaler().fit_transform(pl.array(Yts ).mean(0 ).reshape(-1,1))
Tts[['id', target]].to_csv('text_cat_num_xgb_lgb_NN.csv', index=False )<load_pretrained>
flat_predictions = [item for sublist in preds for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
flat_true_labels = [item for sublist in true for item in sublist]
nltk.download("punkt" )<string_transform>
print(classification_report(flat_predictions,flat_true_labels))
def tokenize_sentences(sentences, words_dict):
tokenized_sentences = []
for sentence in tqdm.tqdm(sentences):
if hasattr(sentence, "decode"):
sentence = sentence.decode("utf-8")
 tokens = nltk.tokenize.word_tokenize(sentence)
 result = []
for word in tokens:
word = word.lower()
if word not in words_dict:
words_dict[word] = len(words_dict)
 word_index = words_dict[word]
result.append(word_index)
 tokenized_sentences.append(result)
 return tokenized_sentences, words_dict<categorify>
comments1 = df_test.text.values
indices1=tokenizer.batch_encode_plus(comments1,max_length=128,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids1=indices1["input_ids"]
attention_masks1=indices1["attention_mask"]
prediction_inputs1= torch.tensor(input_ids1)
 prediction_masks1 = torch.tensor(attention_masks1)
 batch_size = 32
prediction_data1 = TensorDataset(prediction_inputs1, prediction_masks1)
 prediction_sampler1 = SequentialSampler(prediction_data1)
 prediction_dataloader1 = DataLoader(prediction_data1, sampler=prediction_sampler1, batch_size=batch_size )
def read_embedding_list(file_path):
embedding_word_dict = {}
embedding_list = []
f = open(file_path)
 for index, line in enumerate(f):
if index == 0:
continue
values = line.split()
word = values[0]
try:
coefs = np.asarray(values[1:], dtype='float32')
 except:
continue
embedding_list.append(coefs)
 embedding_word_dict[word] = len(embedding_word_dict)
 f.close()
embedding_list = np.array(embedding_list)
 return embedding_list, embedding_word_dict<drop_column>
print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs1)))
 model.eval()
predictions = []
for batch in prediction_dataloader1:
batch = tuple(t.to(device)for t in batch)
 b_input_ids1, b_input_mask1 = batch
with torch.no_grad() :
outputs1 = model(b_input_ids1, token_type_ids=None,
attention_mask=b_input_mask1)
 logits1 = outputs1[0]
logits1 = logits1.detach().cpu().numpy()
predictions.append(logits1)
 flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
<categorify><EOS> 
sample_sub=pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 submit=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':flat_predictions})
 submit.to_csv('submission.csv',index=False )