Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
!pip install transformers
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['jigsaw_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['jigsaw_min'],
concat_sub['jigsaw_median']))
concat_sub[['id', 'toxic']].to_csv('submission5.csv',
index=False, float_format='%.6f' )<define_variables>
nltk.download('stopwords')
 stop_words = stopwords.words('english')
 nlp = spacy.load('en_core_web_sm' )
df1 = [f for f in all_files if '0.9462' in f][0]
df2 = [f for f in all_files if '0.9459b' in f][0]
df3 = [f for f in all_files if '0.9459' in f and '0.9459b' not in f][0]
df4 = [f for f in all_files if '0.9450' in f][0]
df5 = [f for f in all_files if '0.9432' in f][0]
df6 = [f for f in all_files if '0.9428' in f][0]
df7 = [f for f in all_files if '0.9428b' in f][0]
df8 = [f for f in all_files if '0.9416' in f][0]
df9 = [f for f in all_files if '0.9414' in f and '0.9414b' not in f][0]
df10 = [f for f in all_files if '0.9414b' in f][0]
df11 = [f for f in all_files if '0.9409' in f][0]
df12 = [f for f in all_files if '0.9322' in f][0]<load_from_csv>
print(torch.cuda.is_available() )
df1 = pd.read_csv(os.path.join(sub_path, df1))
df2 = pd.read_csv(os.path.join(sub_path, df2))
df3 = pd.read_csv(os.path.join(sub_path, df3))
df4 = pd.read_csv(os.path.join(sub_path, df4))
df5 = pd.read_csv(os.path.join(sub_path, df5))
df6 = pd.read_csv(os.path.join(sub_path, df6))
df7 = pd.read_csv(os.path.join(sub_path, df7))
df8 = pd.read_csv(os.path.join(sub_path, df8))
df9 = pd.read_csv(os.path.join(sub_path, df9))
df10 = pd.read_csv(os.path.join(sub_path, df10))
df11 = pd.read_csv(os.path.join(sub_path, df11))
df12 = pd.read_csv(os.path.join(sub_path, df12))
df1.head()<prepare_output>
def stratified_cv(classifier, X_train, y_train, **kwargs):

 skf = StratifiedKFold(**kwargs)
 total_accuracy = []
total_precision = []
total_recall = []
total_f1 = []
for train_index, val_index in skf.split(X_train, y_train):
current_X_train = X_train[train_index]
current_y_train = y_train.iloc[train_index]
current_X_val = X_train[val_index]
current_y_val = y_train.iloc[val_index]
clf = copy.deepcopy(classifier)
 clf.fit(current_X_train, current_y_train)
 current_predictions = clf.predict(current_X_val)
 total_accuracy.append(accuracy_score(current_y_val, current_predictions))
total_precision.append(precision_score(current_y_val, current_predictions))
total_recall.append(recall_score(current_y_val, current_predictions))
total_f1.append(f1_score(current_y_val, current_predictions))
ave_accuracy = mean(total_accuracy)
 ave_precision = mean(total_precision)
 ave_recall = mean(total_recall)
 ave_f1 = mean(total_f1)
 print("Average Accuracy: {:.4f}".format(ave_accuracy))
print("Average Precision: {:.4f}".format(ave_precision))
print("Average Recall: {:.4f}".format(ave_recall))
print("Average F1: {:.4f}".format(ave_f1))
sub = df1.copy() [['id']]<categorify>
def clean_text(text):
text = re.sub(r"\x89Û_", "", text)
 text = re.sub(r"\x89ÛÒ", "", text)
 text = re.sub(r"\x89ÛÓ", "", text)
 text = re.sub(r"\x89ÛÏWhen", "When", text)
 text = re.sub(r"\x89ÛÏ", "", text)
 text = re.sub(r"China\x89Ûªs", "China's", text)
 text = re.sub(r"let\x89Ûªs", "let's", text)
 text = re.sub(r"\x89Û÷", "", text)
 text = re.sub(r"\x89Ûª", "", text)
 text = re.sub(r"\x89Û\x9d", "", text)
 text = re.sub(r"å_", "", text)
 text = re.sub(r"\x89Û¢", "", text)
 text = re.sub(r"\x89Û¢åÊ", "", text)
 text = re.sub(r"fromåÊwounds", "from wounds", text)
 text = re.sub(r"åÊ", "", text)
 text = re.sub(r"åÈ", "", text)
 text = re.sub(r"JapÌ_n", "Japan", text)
 text = re.sub(r"Ì©", "e", text)
 text = re.sub(r"å¨", "", text)
 text = re.sub(r"SuruÌ¤", "Suruc", text)
 text = re.sub(r"åÇ", "", text)
 text = re.sub(r"å£3million", "3 million", text)
 text = re.sub(r"åÀ", "", text)
 text = re.sub(r'https?://\S+|www\.\S+', '', text)
 text = re.sub(r'<.*?>', '', text)
 text = re.sub(r'[\d]+', ' ', text)
 return text
def preprocess(text, allowed_tags=["N", "J", "R", "V"]):
tag_map = defaultdict(lambda : wn.NOUN)
 tag_map['N'] = wn.NOUN
tag_map['J'] = wn.ADJ
tag_map['R'] = wn.ADV
tag_map['V'] = wn.VERB
text = text.lower()
tokens = [token for token in wordpunct_tokenize(text)]
lemmatizer = wn_lemmatizer()
lemmas = [lemmatizer.lemmatize(token, tag_map[tag[0]])for token, tag in pos_tag(tokens)if tag[0] in allowed_tags]
lemmas = [lemma for lemma in lemmas if lemma not in stop_words and lemma.isalpha() ]
return ' '.join(lemmas )
W = 0.9462 + 2*0.9459 + 0.9450 + 2*0.9428 + 0.9422 + 0.9416 + 2*0.9414 + 0.9409 + 0.9322
w1 = 0.9462/W
w2 = 0.9459/W
w3 = 0.9459/W
w4 = 0.9450/W
w5 = 0.9432/W
w6 = 0.9428/W
w7 = 0.9428/W
w8 = 0.9416/W
w9 = 0.9414/W
w10 = 0.9414/W
w11 = 0.9409/W
w12 = 0.9322/W<feature_engineering>
def array_equal(array_1, array_2):

 if array_1.shape != array_2.shape:
return False
for i in range(len(array_1)) :
if(array_1[i] is array_2[i])or(array_1[i] == array_2[i]):
continue
return False
return True
sub['toxic'] =(w1*df1['toxic'] +
w2*df2['toxic'] +
w3*df3['toxic'] +
w4*df4['toxic'] +
w5*df5['toxic'] +
w6*df6['toxic'] +
w7*df7['toxic'] +
w8*df8['toxic'] +
w9*df9['toxic'] +
w10*df10['toxic'] +
w11*df11['toxic'] +
w12*df12['toxic'])
 sub.head()<save_to_csv>
data = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 data.head()
sub.to_csv('submission_blend.csv', index=False)
<split>
unique_targets = data.groupby('text' ).agg(unique_target=('target', pd.Series.nunique))
controversial_tweets = unique_targets[unique_targets['unique_target'] > 1].index
data = data[~data['text'].isin(controversial_tweets)]
data = data.drop_duplicates(subset='text', keep='first')
 data['text'] = data['text'].apply(clean_text)
 test['text'] = test['text'].apply(clean_text)
 data['value'] = 1
data['keyword'] = data['keyword'].fillna("nan")
 test['value'] = 1
test['keyword'] = test['keyword'].fillna("nan")
 data = pd.pivot_table(data, values="value", index=["id", "text", "target"], columns=["keyword"] ).fillna(0 ).reset_index()
test = pd.pivot_table(test, values="value", index=["id", "text"], columns=["keyword"] ).fillna(0 ).reset_index()
def get_dist_preds(predictions, metric):
new_preds = []
for j, pred in enumerate(predictions):
distances = []
remaining_preds = predictions[:j] + predictions[j+1:]
for pred_ in remaining_preds:
if metric == 'euclid':
distances += [euclidean(pred_, pred)]
elif metric == 'cosine':
distances += [cosine(pred_, pred)]
elif metric == 'jaccard':
distances += [jaccard(pred_, pred)]
elif metric == 'chebyshev':
distances += [chebyshev(pred_, pred)]
elif metric == 'correlation':
distances += [correlation(pred_, pred)]
elif metric == 'cityblock':
distances += [cityblock(pred_, pred)]
elif metric == 'canberra':
distances += [canberra(pred_, pred)]
elif metric == 'braycurtis':
distances += [braycurtis(pred_, pred)]
elif metric == 'hamming':
distances += [hamming(pred_, pred)]
elif metric == 'battacharyya':
distances += [DistanceMetrics.battacharyya(pred_, pred, method='continuous')]
new_preds += [(pred, sum(distances)) ]
weights = [tup[1] for tup in new_preds]
W = sum(weights)
 preds_ascending_dist = sorted(new_preds, key=lambda x: x[1])
 weights_descending = sorted(weights, reverse=True)
 weighted_pred = sum([pred_tup[0]*(weights_descending[k]/W)for k, pred_tup in enumerate(preds_ascending_dist)])
 return weighted_pred<define_variables>
TRAIN_VAL = 0.8
train = data.sample(frac=TRAIN_VAL, random_state=RANDOM_STATE)
 val = data.drop(train.index ).reset_index(drop=True)
 train = train.reset_index(drop=True )
dfs = [df1, df2, df3, df4, df5, df6, df7, df8, df9, df10, df11, df12]
preds = [np.array(df['toxic'])for df in dfs]<predict_on_test>
train_pruned = train.copy(deep=True)
 train_pruned['text'] = train_pruned['text'].progress_apply(preprocess)
 val_pruned = val.copy(deep=True)
 val_pruned['text'] = val_pruned['text'].progress_apply(preprocess)
 test_pruned = test.copy(deep=True)
 test_pruned['text'] = test_pruned['text'].progress_apply(preprocess )
df_deboost = get_dist_preds(preds, 'correlation' )<prepare_output>
all_text = pd.concat([train_pruned.drop(['target'], axis=1), val_pruned.drop(['target'], axis=1), test_pruned], ignore_index=True )
_sub = df1.copy() [['id']]
_sub['toxic'] = df_deboost
_sub.head()<save_to_csv>
cv = CountVectorizer(ngram_range=(1,2))
tfidf_transformer = TfidfTransformer()
counts = cv.fit_transform(all_text['text'])
 tfidf = tfidf_transformer.fit_transform(counts )
_sub.to_csv('submission_deboost.csv', index=False)
<define_variables>
augmented = hstack([tfidf, csr_matrix(all_text.iloc[:, 3:])] ).tocsr()
n_folds = len(dfs)
 n_folds<define_variables>
train_size = train_pruned.shape[0]
val_size = val_pruned.shape[0]
X_train_tfidf = tfidf[:train_size,:]
X_train_augmented = augmented[:train_size,:]
X_val_tfidf = tfidf[train_size: val_size + train_size,:]
X_val_augmented = augmented[train_size: val_size + train_size,:]
X_test_tfidf = tfidf[val_size + train_size:,:]
X_test_augmented = augmented[val_size + train_size:,:]
nrow = len(dfs[0])
 nrow<define_variables>
y_train_tfidf = train_pruned['target']
y_val_tfidf = val_pruned['target']
num_row_per_df = nrow//n_folds
num_row_per_df<define_variables>
X_train_keyword = csr_matrix(train_pruned.iloc[:, 3:] ).tocsr()
X_val_keyword = csr_matrix(val_pruned.iloc[:, 3:] ).tocsr()
X_test_keyword = csr_matrix(test_pruned.iloc[:, 2:] ).tocsr()
remainder_rows = nrow - n_folds * num_row_per_df
remainder_rows<define_variables>
mb_clf = MB()
mb_clf.fit(X_train_tfidf, y_train_tfidf)
 pred = mb_clf.predict(X_train_tfidf)
 print(classification_report(y_train_tfidf, pred))
stratified_cv(mb_clf, X_train_tfidf, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )
end_indices = [num_row_per_df*i for i in range(1,n_folds)] + [nrow]
end_indices<define_variables>
mb_clf_augmented = MB()
mb_clf_augmented.fit(X_train_augmented, y_train_tfidf)
 pred = mb_clf_augmented.predict(X_train_augmented)
 print(classification_report(y_train_tfidf,pred))
stratified_cv(mb_clf_augmented, X_train_augmented, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )
partitions = [[[]]*n_folds]*n_folds
partitions<feature_engineering>
mb_clf_keyword = MB()
mb_clf_keyword.fit(X_train_keyword, y_train_tfidf)
 pred = mb_clf_keyword.predict(X_train_keyword)
 print(classification_report(y_train_tfidf,pred))
stratified_cv(mb_clf_keyword, X_train_keyword, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )
data = {}
for i, df in enumerate(dfs):
data[str(i)] = df
ranks = pd.DataFrame(columns=data.keys())
 for key in data.keys() :
ranks[key] = data[key].toxic.rank(method='min')
 ranks['Average'] = ranks.mean(axis=1)
 ranks['Scaled Rank'] =(ranks['Average'] - ranks['Average'].min())/(ranks['Average'].max() - ranks['Average'].min())
 ranks.corr() [:1]<save_to_csv>
weights = [0.15, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.05, 0.05, 0.05, 0.05, 0.05]
ranks['Score'] = ranks[[str(i)for i in range(n_folds)]].mul(weights ).sum(1)/ ranks.shape[0]
submission_lb = pd.read_csv(".. /input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv")
 submission_lb['toxic'] = ranks['Score']
submission_lb.to_csv("WEIGHT_AVERAGE_RANK.csv", index=False)
 submission_lb.head()<compute_test_metric>
svm_clf = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')
 svm_clf.fit(X_train_tfidf, y_train_tfidf)
 pred = svm_clf.predict(X_train_tfidf)
 print(classification_report(y_train_tfidf,pred))
stratified_cv(svm_clf, X_train_tfidf, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )
labels = ["toxic"]
for label in labels:
print(label)
 print(np.round(np.corrcoef([df[label].rank(pct=True)for df in dfs]), 4))<define_variables>
svm_clf_augmented = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')
 svm_clf_augmented.fit(X_train_augmented, y_train_tfidf)
 pred = svm_clf_augmented.predict(X_train_augmented)
 print(classification_report(y_train_tfidf,pred))
stratified_cv(svm_clf_augmented, X_train_augmented, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )
submission = dfs[0][['id']]
for label in labels:
preds = []
submission[label] =(
 0.15*(0.9*dfs[0][label] + 0.1*dfs[5][label])+
0.15*(0.9*dfs[1][label] + 0.1*dfs[5][label])+
0.15*(0.9*dfs[2][label] + 0.1*dfs[7][label])+
0.15*(0.9*dfs[3][label] + 0.1*dfs[7][label])+
0.1*(0.5*dfs[4][label] + 0.5*dfs[5][label])+
0.1*(0.5*dfs[5][label] + 0.5*dfs[4][label])+
0.1*(0.1*dfs[6][label] + 0.9*dfs[4][label])+
0.1*(0.5*dfs[7][label] + 0.5*dfs[5][label])
 )<save_to_csv>
svm_clf_keyword = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')
 svm_clf_keyword.fit(X_train_keyword, y_train_tfidf)
 pred = svm_clf_keyword.predict(X_train_keyword)
 print(classification_report(y_train_tfidf,pred))
stratified_cv(svm_clf_keyword, X_train_keyword, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )
submission.to_csv('submission_correlations.csv', index=False)
<define_variables>
svm_train_preds = svm_clf_keyword.predict(X_train_keyword)
 svm_val_preds = svm_clf_keyword.predict(X_val_keyword)
 svm_test_preds = svm_clf_keyword.predict(X_test_keyword )
df1 = [f for f in all_files if '0.9462' in f][0]
df2 = [f for f in all_files if '0.9459b' in f][0]
df3 = [f for f in all_files if '0.9459' in f and '0.9459b' not in f][0]
df4 = [f for f in all_files if '0.9450' in f][0]
df5 = [f for f in all_files if '0.9432' in f][0]
df6 = [f for f in all_files if '0.9428b' in f][0]
df7 = [f for f in all_files if '0.9428' in f][0]
df8 = [f for f in all_files if '0.9416' in f][0]
df9 = [f for f in all_files if '0.9414' in f][0]
df10 = [f for f in all_files if '0.9322' in f][0]
df1 = pd.read_csv(os.path.join(sub_path, df1))
df2 = pd.read_csv(os.path.join(sub_path, df2))
df3 = pd.read_csv(os.path.join(sub_path, df3))
df4 = pd.read_csv(os.path.join(sub_path, df4))
df5 = pd.read_csv(os.path.join(sub_path, df5))
df6 = pd.read_csv(os.path.join(sub_path, df6))
df7 = pd.read_csv(os.path.join(sub_path, df7))
df8 = pd.read_csv(os.path.join(sub_path, df8))
df9 = pd.read_csv(os.path.join(sub_path, df9))
df10 = pd.read_csv(os.path.join(sub_path, df10))<define_variables>
rf_clf = RandomForestClassifier(n_estimators = 100, random_state = RANDOM_STATE)
 rf_clf.fit(X_train_tfidf, y_train_tfidf)
 rf_preds = rf_clf.predict(X_train_tfidf)
 print(classification_report(y_train_tfidf, rf_preds))
stratified_cv(rf_clf, X_train_tfidf, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )
dfs = [df1, df2, df3, df4, df5, df6, df7, df8, df9, df10]<init_hyperparams>
rf_clf_augmented = RandomForestClassifier(n_estimators = 100, random_state = RANDOM_STATE)
 rf_clf_augmented.fit(X_train_augmented, y_train_tfidf)
 rf_preds_augmented = rf_clf_augmented.predict(X_train_augmented)
 print(classification_report(y_train_tfidf, rf_preds))
stratified_cv(rf_clf_augmented, X_train_augmented, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )
p = 1.5<compute_test_metric>
rf_train_preds = rf_clf_augmented.predict(X_train_augmented)
 rf_val_preds = rf_clf_augmented.predict(X_val_augmented)
 rf_test_preds = rf_clf_augmented.predict(X_test_augmented )
labels = ["toxic"]
for label in labels:
print(label)
 print(np.round(np.corrcoef([df[label].rank(pct=True)for df in dfs]), 4))<prepare_output>
lr_clf = LogisticRegression(fit_intercept=True,
random_state=RANDOM_STATE,
solver = 'liblinear',
penalty = 'l1',
max_iter = 200)
 lr_clf.fit(X_train_tfidf, y_train_tfidf)
 lr_preds = lr_clf.predict(X_train_tfidf)
 print(classification_report(y_train_tfidf, lr_preds))
stratified_cv(lr_clf, X_train_tfidf, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )
sub_power = submission[['id']].copy()
sub_power['toxic'] =(df1['toxic']**p * 0.2 +
df2['toxic']**p * 0.1 +
df3['toxic']**p * 0.1 +
df4['toxic']**p * 0.15 +
df5['toxic']**p * 0.25 +
df6['toxic']**p * 0.025 +
df7['toxic']**p * 0.025 +
df8['toxic']**p * 0.05 +
df9['toxic']**p * 0.05 +
df10['toxic']**p * 0.05 )<save_to_csv>
lr_clf = LogisticRegression(fit_intercept=True,
random_state=RANDOM_STATE,
solver = 'liblinear',
penalty = 'l1',
max_iter = 200)
 lr_clf.fit(X_train_augmented, y_train_tfidf)
 lr_preds = lr_clf.predict(X_train_augmented)
 print(classification_report(y_train_tfidf, lr_preds))
stratified_cv(lr_clf, X_train_augmented, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )
sub_power.to_csv('submission_power_ensemble.csv', index=False)
<prepare_output>
tf.config.set_visible_devices([], 'GPU')
 visible_devices = tf.config.get_visible_devices()
sub = submission[['id']].copy()<prepare_output>
train_corpus = train_pruned['text'].apply(lambda x: x.split() ).to_list()
val_corpus = val_pruned['text'].apply(lambda x: x.split() ).to_list()
test_corpus = test_pruned['text'].apply(lambda x: x.split() ).to_list()
sub['toxic'] = 0.1*df10['toxic'] + 0.9*df9['toxic']
sub['toxic'] = 0.51*df9['toxic'] + 0.49*sub['toxic']
sub['toxic_temp'] = 0.1*df8['toxic'] + 0.9*df7['toxic']
sub['toxic_temp'] = 0.51*df7['toxic'] + 0.49*sub['toxic_temp']
sub['toxic'] = 0.5*sub['toxic_temp'] + 0.5*sub['toxic']
sub['toxic_temp'] = 0.1*df6['toxic'] + 0.9*df5['toxic']
sub['toxic_temp'] = 0.51*df5['toxic'] + 0.49*sub['toxic_temp']
sub['toxic'] = 0.5*sub['toxic_temp'] + 0.5*sub['toxic']
sub['toxic_temp'] = 0.1*df6['toxic'] + 0.9*df5['toxic']
sub['toxic_temp'] = 0.51*df5['toxic'] + 0.49*sub['toxic_temp']
sub['toxic'] = 0.5*sub['toxic_temp'] + 0.5*sub['toxic']
sub['toxic_temp'] = 0.1*df4['toxic'] + 0.9*df3['toxic']
sub['toxic_temp'] = 0.51*df3['toxic'] + 0.49*sub['toxic_temp']
sub['toxic'] = 0.5*sub['toxic_temp'] + 0.5*sub['toxic']
sub['toxic_temp'] = 0.1*df2['toxic'] + 0.9*df1['toxic']
sub['toxic_temp'] = 0.51*df1['toxic'] + 0.49*sub['toxic_temp']
sub['toxic'] = 0.5*sub['toxic_temp'] + 0.5*sub['toxic']<save_to_csv>
corpus = train_corpus + val_corpus + test_corpus
sub[['id', 'toxic']].to_csv('submission_pairwise_blend_ensemble.csv', index=False)
 sub[['id', 'toxic']].head()<prepare_output>
tokenizer = Tokenizer()
tokenizer.fit_on_texts(corpus)
 train_sequences = tokenizer.texts_to_sequences(train_corpus)
 val_sequences = tokenizer.texts_to_sequences(val_corpus)
 test_sequences = tokenizer.texts_to_sequences(test_corpus)
 word_index = tokenizer.word_index
train_max_length = max([len(x)for x in train_sequences])
 val_max_length = max([len(x)for x in val_sequences])
 test_max_length = max([len(x)for x in test_sequences])
 max_length = max(train_max_length, val_max_length, test_max_length)
 X_train_pad = pad_sequences(train_sequences, maxlen=max_length, padding="post")
 y_train = train_pruned['target']
X_val_pad = pad_sequences(val_sequences, maxlen=max_length, padding="post")
 y_val = val_pruned['target']
X_test_pad = pad_sequences(test_sequences, maxlen=max_length, padding="post")
 vocab = np.array(list(tokenizer.word_index.keys()))
vocab_size = len(tokenizer.word_index)+ 1
sub = sub[['id']].copy()
sub_final = sub[['id']].copy()
sub['toxic'] = concat_sub['jigsaw_mean']
sub.head()<define_variables>
EMBEDDING_DIM = 30
upper = 0.95
lower = 0.05
lp = 0.8
up = 1.5<feature_engineering>
def ConvNet(max_sequence_length, num_words, embedding_dim, labels_index):
embedding_layer = Embedding(num_words,
embedding_dim,
input_length=max_sequence_length)
 sequence_input = Input(shape=(max_sequence_length,), dtype='int32')
 embedded_sequences = embedding_layer(sequence_input)
 print()
convs = []
filter_sizes = [2, 3, 4, 5, 6]
for filter_size in filter_sizes:
l_conv = Conv1D(filters=30,
kernel_size=filter_size,
activation='relu' )(embedded_sequences)
 l_conv = Dropout(0.2 )(l_conv)
 l_pool = GlobalMaxPooling1D()(l_conv)
 convs.append(l_pool)
 l_merge = concatenate(convs, axis=1)
 x = Dense(30, activation='relu' )(l_merge)
 x = Dropout(0.2 )(x)
 preds = Dense(labels_index, activation='sigmoid', kernel_regularizer=l1_l2(0.01, 0.01))(x)
 model = Model(sequence_input, preds)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['acc', Precision() , Recall() ])
 model.summary()
return model
sub[(sub['toxic'] <= lower)] = sub[(sub['toxic'] <= lower)]**lp 
<prepare_output>
cnn_model = ConvNet(max_length, vocab_size, EMBEDDING_DIM, 1 )
sub_final['toxic'] = sub['toxic']
sub_final.head()<save_to_csv>
hist = cnn_model.fit(X_train_pad,
y_train,
epochs=10,
batch_size=64,
validation_data=(X_val_pad, y_val),
verbose=2 )
sub_final.to_csv('submission_minmax_power_scaling_ensemble.csv', index=False)
 sub_final.head()<import_modules>
cnn_test_preds = cnn_model.predict(X_test_pad)
 cnn_test_preds_binary = list(map(lambda x: 1 if x >= 0.5 else 0, cnn_test_preds))
cnn_train_preds = cnn_model.predict(X_train_pad)
 cnn_train_preds_binary = list(map(lambda x: 1 if x >= 0.5 else 0, cnn_train_preds))
cnn_val_preds = cnn_model.predict(X_val_pad)
 cnn_val_preds_binary = list(map(lambda x: 1 if x >= 0.5 else 0, cnn_val_preds))
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
import time
import os
import re
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input,GRU
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras import optimizers
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<load_from_csv>
def align_data() :
en=pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')
 f1=pd.read_csv('.. /input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')
 result=[]
last_cell=[]
for i in tqdm(range(len(en['id']))):
index=np.where(f1['id']==en['id'][i])[0]
l=len(index)
 if l==1:
cell=[f1['comment_text'][index[0]],'es',f1['toxic'][index[0]]]
result.append(cell)
 last_cell=cell
else:
cell=last_cell
result.append(cell)
 print(len(en['toxic']))
print(len(result))
column=['comment_text','lang','toxic']
toxic=pd.DataFrame(columns=column,data=result)
 toxic.to_csv('/kaggle/working/train1_es_align.csv'%(LANG,LANG),index=False )<load_from_csv>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print("GPU is available: {}".format(torch.cuda.is_available()))
def mix_data() :
ES_FRAC=0.132232182034727
TR_FRAC=0.21939447125932426
RU_FRAC=0.17156647652479157
IT_FRAC=0.13310975991976431
FR_FRAC=0.17112768758227292
PT_FRAC=0.17256942267911993
train_tr=pd.read_csv('.. /input/jigsaw_toxic_comment_train_align/train1_tr.csv.csv')
 train_es=pd.read_csv('.. /input/jigsaw_toxic_comment_train_align/train1_es.csv.csv')
 train_it=pd.read_csv('.. /input/jigsaw_toxic_comment_train_align/train1_it.csv.csv')
 train_fr=pd.read_csv('.. /input/jigsaw_toxic_comment_train_align/train1_fr.csv.csv')
 train_ru=pd.read_csv('.. /input/jigsaw_toxic_comment_train_align/train1_ru.csv.csv')
 train_pt=pd.read_csv('.. /input/jigsaw_toxic_comment_train_align/train1_pt.csv.csv')
 result=[]
label=[]
for i in tqdm(range(0, len(train_es['comment_text']))):
rf = random.random()
if rf < ES_FRAC:
cell = [train_es['comment_text'][i], train_es['lang'][i], train_es['toxic'][i]]
label.append(cell[2])
 result.append(cell)
 elif rf <(ES_FRAC + FR_FRAC):
cell = [train_fr['comment_text'][i], train_fr['lang'][i], train_fr['toxic'][i]]
label.append(cell[2])
 result.append(cell)
 elif rf <(ES_FRAC + FR_FRAC + TR_FRAC):
cell = [train_tr['comment_text'][i], train_tr['lang'][i], train_tr['toxic'][i]]
label.append(cell[2])
 result.append(cell)
 elif rf <(ES_FRAC + FR_FRAC + TR_FRAC + IT_FRAC):
cell = [train_it['comment_text'][i], train_it['lang'][i], train_it['toxic'][i]]
label.append(cell[2])
 result.append(cell)
 elif rf <(ES_FRAC + FR_FRAC + TR_FRAC + IT_FRAC + PT_FRAC):
cell = [train_pt['comment_text'][i], train_pt['lang'][i], train_pt['toxic'][i]]
label.append(cell[2])
 result.append(cell)
 else:
cell = [train_ru['comment_text'][i], train_ru['lang'][i], train_ru['toxic'][i]]
label.append(cell[2])
 result.append(cell)
 print(len(result))
print(len(label))
label=np.array(label)
 np.save('/kaggle/working/train1_label.npy')
 column=['comment_text','lang','toxic']
toxic=pd.DataFrame(columns=column,data=result)
 toxic.to_csv('/kaggle/working/train1_mix.csv')
 x_train=regular_encode(toxic['comment_text'], tokenizer, maxlen=MAX_LEN)
 np.save('/kaggle/working/train1_mix.npy' )<choose_model_class>
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
 model = RobertaForSequenceClassification.from_pretrained('roberta-base' ).to(device )
def build_model(transformer, max_len=512,learnig_rate=1e-5):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=learnig_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model<define_variables>
def prepare_features(data_set, labels=None, max_seq_length = 200,
zero_pad = True, include_special_tokens = True):
input_ids = []
attention_masks = []
for sent in data_set:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = include_special_tokens,
max_length = max_seq_length,
pad_to_max_length = zero_pad,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 if labels is not None:
labels = torch.tensor(labels)
 return input_ids, attention_masks, labels
else:
return input_ids, attention_masks
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path("jigsaw-multilingual-toxic-comment-classification")
 EPOCHS = 3
BATCH_SIZE = 24 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'<load_from_csv>
BATCH_SIZE = 32
LEARNING_RATE = 1e-05
EPSILON = 1e-8
MAX_EPOCHS = 5
valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<train_model>
train_val = 0.8
train = data.sample(frac=train_val, random_state=RANDOM_STATE)
 val = data.drop(train.index ).reset_index(drop=True)
 train = train.reset_index(drop=True)
 train_input_ids, train_attention_masks, train_labels = prepare_features(
 train['text'], train['target'])
 val_input_ids, val_attention_masks, val_labels = prepare_features(
 val['text'], val['target'])
 test_input_ids, test_attention_masks = prepare_features(
 test['text'])
 training_set = TensorDataset(train_input_ids, train_attention_masks, train_labels)
 validation_set = TensorDataset(val_input_ids, val_attention_masks, val_labels)
 test_set = TensorDataset(test_input_ids, test_attention_masks)
 loading_params = {'batch_size': BATCH_SIZE,
'shuffle': True,
'drop_last': False,
'num_workers': 1}
loading_params_no_shuffle = {'batch_size': BATCH_SIZE,
'shuffle': False,
'drop_last': False,
'num_workers': 1}
training_loader = DataLoader(training_set, **loading_params)
 validation_loader = DataLoader(validation_set, **loading_params)
 training_loader_no_shuffle = DataLoader(training_set, **loading_params_no_shuffle)
 validation_loader_no_shuffle = DataLoader(validation_set, **loading_params_no_shuffle)
 testing_loader = DataLoader(test_set, **loading_params_no_shuffle )
x_valid=np.load('/kaggle/input/toxic-npy/x_valid_encode192.npy')
 print(x_valid)
 print(x_valid.shape)
 y_valid = valid.toxic.values
print(y_valid)
 print(y_valid.shape )<load_pretrained>
optimizer = AdamW(model.parameters() ,
lr = LEARNING_RATE,
eps = EPSILON 
)
 total_steps = len(training_loader)* MAX_EPOCHS
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )
x_test=np.load('/kaggle/input/toxic-npy/x_test_encode192.npy')
 print(x_test)
 print(x_test.shape)
 print("finish x_test_encode" )<categorify>
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))
valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
)
<train_on_grid>
for epoch in tqdm_notebook(range(MAX_EPOCHS)) :
t0 = time.time()
total_train_loss = 0
model.train()
print("EPOCH -- {} / {}".format(epoch, MAX_EPOCHS))
for step, batch in enumerate(training_loader):
if step % 30 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {} of {}.Elapsed: {:}'.format(step, len(training_loader), elapsed))
input_ids = batch[0].to(device ).to(torch.int64)
 input_masks = batch[1].to(device ).to(torch.int64)
 labels = batch[2].to(device ).to(torch.int64)
 model.zero_grad()
loss, logits = model(input_ids,
token_type_ids=None,
attention_mask=input_masks,
labels=labels)
 logits = logits.detach().cpu().numpy()
label_ids = labels.to('cpu' ).numpy()
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(training_loader)
 training_time = format_time(time.time() - t0)
 print('')
 print(' Average training loss: {0:.4f}'.format(avg_train_loss))
print(' Training epoch took: {:}'.format(training_time))
print('Running Validation')
 model.eval()
val_predictions = []
val_labels = []
for batch in validation_loader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 val_predictions.append(logits.detach().cpu().numpy())
 val_labels.append(b_labels.to('cpu' ).numpy())
 val_predictions = np.array([item for sublist in val_predictions for item in sublist])
 val_labels = np.array([item for sublist in val_labels for item in sublist])
 val_predictions_flat = np.argmax(val_predictions, axis=1)
 print(val_predictions_flat.shape)
 val_accuracy = accuracy_score(val_labels, val_predictions_flat)
 val_recall = recall_score(val_labels, val_predictions_flat)
 val_precision = precision_score(val_labels, val_predictions_flat)
 val_f1 = f1_score(val_labels, val_predictions_flat)
 print('\tAccuracy: {0:.4f}
\tRecall: {1:.4f}
\tPrecision: {2:.4f}
\tF1_score: {3:.4f}
\t'.
 format(val_accuracy, val_recall, val_precision, val_f1))
def train_train2max() :
print('train train2max')
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 x_train=np.load('/kaggle/input/mix-data2/train2max21.npy')
 y_train=np.load('/kaggle/input/mix-data2/train2max_label21.npy')
 y_train=y_train.round().astype(int)
 train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.batch(BATCH_SIZE)
.shuffle(2048)
.prefetch(AUTO)
)
 with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN,learnig_rate=5e-6)
 model.summary()
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=1 
)
 model.save_weights('/kaggle/working/model_mix_1.h5' )<train_on_grid>
model.eval()
roberta_train_preds = []
for batch in training_loader_no_shuffle:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 roberta_train_preds.append(logits.detach().cpu().numpy())
 roberta_train_preds = np.array([item for sublist in roberta_train_preds for item in sublist])
 roberta_val_preds = []
for batch in validation_loader_no_shuffle:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 roberta_val_preds.append(logits.detach().cpu().numpy())
 roberta_val_preds = np.array([item for sublist in roberta_val_preds for item in sublist])
 roberta_test_preds = []
for batch in testing_loader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 with torch.no_grad() :
logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)[0]
roberta_test_preds.append(logits.detach().cpu().numpy())
 roberta_test_preds = np.array([item for sublist in roberta_test_preds for item in sublist])
 roberta_train_preds_binary = np.argmax(roberta_train_preds, axis=1)
 roberta_val_preds_binary = np.argmax(roberta_val_preds, axis=1)
 roberta_test_preds_binary = np.argmax(roberta_test_preds, axis=1)
 roberta_train_preds = pd.DataFrame(roberta_train_preds, columns=["roberta_fake","roberta_real"])
 roberta_val_preds = pd.DataFrame(roberta_val_preds, columns=["roberta_fake","roberta_real"])
 roberta_test_preds = pd.DataFrame(roberta_test_preds, columns=["roberta_fake","roberta_real"] )
def train_train1() :
print('train train1')
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 x_train=np.load('/kaggle/input/mix-data2/train1_mix_data21.npy')
 y_train=np.load('/kaggle/input/mix-data2/train1_label21.npy')
 print(y_train)
 print(y_train.shape)
 train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.batch(BATCH_SIZE)
.shuffle(2048)
.prefetch(AUTO)
)
 with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN,learnig_rate=5e-6)
 model.load_weights('/kaggle/working/model_mix_1.h5')
 print("learning rate:5e-6")
 n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=1 
)
 model.save_weights('/kaggle/working/model_mix_1.h5' )<define_search_model>
train_stacking = train.loc[:, ["target"]]
val_stacking = val.loc[:, ["target"]]
test_stacking = test.loc[:, []]
train_stacking["svm_predicted"] = svm_train_preds
train_stacking = pd.concat([train_stacking, roberta_train_preds], axis=1)
 X_train_stacking = train_stacking.drop(["target"], axis=1)
 y_train_stacking = train_stacking["target"]
val_stacking["svm_predicted"] = svm_val_preds
val_stacking = pd.concat([val_stacking, roberta_val_preds], axis=1)
 X_val_stacking = val_stacking.drop(["target"], axis=1)
 y_val_stacking = val_stacking["target"]
test_stacking["svm_predicted"] = svm_test_preds
test_stacking = pd.concat([test_stacking, roberta_test_preds], axis=1)
 X_test_stacking = test_stacking
def train_valid() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN,learnig_rate=4e-6)
 model.load_weights('/kaggle/working/model_mix_1.h5')
 n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=2
)<save_to_csv>
svm_clf_stacking = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')
 svm_clf_stacking.fit(X_train_stacking, y_train_stacking)
 pred = svm_clf_stacking.predict(X_val_stacking)
 print(f1_score(y_val_stacking,pred))
def predict() :
score = model.predict(test_dataset,verbose=1)
 sub['toxic']=score[:sub['toxic'].shape[0]]
sub.to_csv('submission.csv', index=False )<load_from_csv>
lr_clf_stacking = LogisticRegression(fit_intercept=True,
random_state=RANDOM_STATE,
solver = 'liblinear',
penalty = 'l1',
max_iter = 200)
 lr_clf_stacking.fit(X_train_stacking, y_train_stacking)
 lr_preds = lr_clf_stacking.predict(X_train_stacking)
 print(accuracy_score(y_train_stacking,lr_preds))
submission_train2mix1 = pd.read_csv('/kaggle/input/haveatry/submission9450.csv')
 submission_train2mix2 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_2.csv')
 submission_train2mix3 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_3.csv')
 submission_train2mix4 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_4.csv')
 submission_train2mix5 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_5.csv')
 submission_train2mix6 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_6.csv')
 submission_train2mix7 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_7.csv')
 submission_train2mix8 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_8.csv')
 submission_train2mix9 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_9.csv')
 submission_train2mix10 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_10.csv')
 submission_train2mix11 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_11.csv')
 submission_train2mix12 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_12.csv')
 submission_train2mix13 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_13.csv')
 submission_train2mix14 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_14.csv')
 submission_train2mix15 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_15.csv')
 submission_train2mix16 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_16.csv')
 submission_train2mix17 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_17.csv')
 submission_train2mix18 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_18.csv')
 submission_train2mix19 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_19.csv')
 submission_train2mix20 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_20.csv')
 submission_train2mix21 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_21.csv' )<prepare_output>
rf_clf = RandomForestClassifier(n_estimators = 100, max_depth=2, random_state = RANDOM_STATE)
 rf_clf.fit(X_train_stacking, y_train_stacking)
 rf_preds = rf_clf.predict(X_train_stacking)
 print(accuracy_score(y_train_stacking, rf_preds))
for i in range(len(submission_train2mix1['toxic'])) :
sub['toxic'][i]=(submission_train2mix1['toxic'][i]+submission_train2mix2['toxic'][i]
+submission_train2mix3['toxic'][i]+submission_train2mix4['toxic'][i]
+submission_train2mix5['toxic'][i]+submission_train2mix6['toxic'][i]
+submission_train2mix7['toxic'][i]+submission_train2mix8['toxic'][i]
+submission_train2mix9['toxic'][i]+submission_train2mix10['toxic'][i]
+submission_train2mix11['toxic'][i]+submission_train2mix12['toxic'][i]
+submission_train2mix13['toxic'][i]+submission_train2mix14['toxic'][i]
+submission_train2mix15['toxic'][i]+submission_train2mix16['toxic'][i]
+submission_train2mix17['toxic'][i]+submission_train2mix18['toxic'][i]
+submission_train2mix19['toxic'][i]+submission_train2mix20['toxic'][i]
+submission_train2mix21['toxic'][i])/21 
<save_to_csv>
test["target"] = lr_clf_stacking.predict(X_test_stacking )
<load_from_csv><EOS> 
test[['id', 'target']].to_csv("submission_stacking.csv", index=False )