Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<find_best_params>
%matplotlib inline
%load_ext autoreload
%autoreload 2
!pip install wordcloud
!pip install transformers==3.0.2
!pip install simpletransformers
!pip install sklearn
!pip install nltk
!pip install unidecode
!pip install normalise
!pip install contractions
sys.path.insert(1, '/kaggle/input/pymodules4/')
 os.system('python3 -m spacy download en')
 train_df = pd.read_csv("/kaggle/input/data-baby2/train.csv")
 test_df = pd.read_csv("/kaggle/input/data-baby2/test.csv")
 msk = np.random.rand(len(train_df)) < 0.3
dev_df = train_df[msk]
train_df = train_df[~msk]
dev_df.reset_index(inplace=True)
 train_df.reset_index(inplace=True )
dog_bytes['Group'] = cluster.KMeans(n_clusters=400, random_state=4, n_jobs=-1 ).fit_predict(dog_bytes[list(range(30)) ])
 dog_bytes['Group'].value_counts() [:5]<define_variables>
train_df[train_df["target"] == 0]["text"].values[1]
fig = plt.figure(figsize=(8, 80))
samples = []
for i in range(400):
g = dog_bytes[dog_bytes['Group'] == i]
if len(g)> 5:
samples += list(g['Path'].values[:5])
 for i in range(len(samples)) [:50]:
ax = fig.add_subplot(len(samples)/5, 5, i+1, xticks=[], yticks=[])
 img = Image.open(samples[i])
 img = img.resize(( 100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)
 img = img.crop(( 0, 0, 64, 64))
plt.imshow(img )<categorify>
train_has_keyword_df = train_df[train_df.keyword.notnull() ]
train_has_location_df = train_df[train_df.location.notnull() ]
train_has_keyword_df.head()
def sim_img(path):
img = Image.open(path ).convert('RGB')
 img = img.resize(( 100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)
 img = img.crop(( 0, 0, 64, 64))
return img
samples = []
for i in range(400):
g = dog_bytes[dog_bytes['Group'] == i]
p = g['Path'].values
for i in range(0,len(p)-2, 2):
samples.append([p[i],p[i+1]])
 if len(samples)> 11_000: break
for i in range(0,len(samples)-1):
samples.append([samples[i][0],samples[i+1][0]])
 if len(samples)> 11_000: break
print(len(samples))<feature_engineering>
text = " ".join(keyword for keyword in train_has_keyword_df.keyword)
 print("There are {} words in the combination of all keywords.".format(len(text)) )
def get_features(path):
try:
st = []
img = Image.open(path)
 im_stats_ = ImageStat.Stat(img)
 st += im_stats_.sum
st += im_stats_.mean
st += im_stats_.rms
st += im_stats_.var
st += im_stats_.stddev
except:
print(path)
 return [path, st]
def normalize_img(paths):
imf_d = {}
p = Pool(cpu_count())
 ret = p.map(get_features, paths)
 for i in range(len(ret)) :
imf_d[ret[i][0]] = ret[i][1]
ret = []
fdata = [imf_d[f] for f in paths]
return pd.DataFrame(fdata)
 dog_bytes = pd.DataFrame(glob.glob('.. /input/all-dogs/all-dogs/**'), columns=['Path'])
 dog_bytes = pd.concat(( dog_bytes, normalize_img(dog_bytes.Path.values)) , axis=1)
 dog_bytes.head()<predict_on_test>
text = " ".join(keyword for keyword in train_has_keyword_df.keyword)
 renderWordcloud(text )
dog_bytes['Group'] = cluster.KMeans(n_clusters=100, random_state=3, n_jobs=-1 ).fit_predict(dog_bytes[list(range(15)) ])
 dog_bytes['Group'].value_counts() [:5]<prepare_x_and_y>
len(train_has_keyword_df.keyword.unique()), len(train_has_location_df.location.unique() )
train = pd.read_csv(".. /input/train.csv" ).drop('id', axis=1)
 y_train = train['target']
X_train = train.drop('target', axis=1)
 test = pd.read_csv('.. /input/test.csv')
 X_test = test.drop('id', axis = 1 )<normalization>
group_keyword_sum_target = train_has_keyword_df.groupby("keyword" ).sum().sort_values("target")
 group_keyword_len = train_has_keyword_df.groupby("keyword" ).count()
group_keyword_sum_target
def winsor_func(s):
return mstats.winsorize(s, limits=[0.00, 0.01])
 def using_mstats(df):
return df.apply(winsor_func, axis=0)
 X_train_winsor=X_train.apply(winsor_func)
 X_test_winsor=X_test.apply(winsor_func )<train_model>
pd.set_option('display.max_rows', None)
 group_keyword_sum_target
clf = LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear' ).fit(X_train_winsor, y_train )<save_to_csv>
group_location_sum_target = train_has_location_df.groupby("location" ).sum().sort_values("target")
 group_location_len = train_has_location_df.groupby("location" ).count()
group_location_sum_target['true/all'] = group_location_sum_target.target / group_location_len.target
y_out=clf.predict_proba(X_test)[:,1]
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = y_out
submission.to_csv('submission_winsor.csv', index=False )<import_modules>
group_location_sum_target.sort_values(by=['target'], ascending=False)[:50]
import pandas as pd
import numpy as np<import_modules>
pd.set_option('display.max_rows', 20 )
import pandas as pd
import numpy as np<load_from_csv>
count_vectorizer = feature_extraction.text.CountVectorizer()
example_train_vectors = count_vectorizer.fit_transform(train_df["text"][0:5] )
df_train = pd.read_csv('.. /input/train.csv', index_col='id')
 df_test = pd.read_csv('.. /input/test.csv', index_col='id')
 df_train.shape, df_test.shape<prepare_x_and_y>
print(example_train_vectors[0].todense().shape)
 print(example_train_vectors[0].todense() )
target = 'target'
X_train = df_train.drop([target], axis=1 ).values
y_train = df_train[target]<import_modules>
train_vectors = count_vectorizer.fit_transform(train_df["text"])
 dev_vectors = count_vectorizer.transform(dev_df["text"] )
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from stability_selection import StabilitySelection<choose_model_class>
clf = linear_model.RidgeClassifier()
Cs = np.logspace(-5, 5, 21)
 pipe = Pipeline(steps=[ 
('scaler', StandardScaler()), 
('clf', LogisticRegression(solver='lbfgs', verbose=1, n_jobs=1, random_state=42))
])
 ss = StabilitySelection(base_estimator=pipe,
lambda_name='clf__C', lambda_grid=Cs,
n_jobs=-1, verbose=1, random_state=42 )<train_model>
%%time
clf.fit(train_vectors, train_df["target"])
 predictions = clf.predict(dev_vectors);
print(classification_report(dev_df['target'], predictions))
%%time
ss.fit(X_train, y_train )<import_modules>
count_vectorizer = feature_extraction.text.CountVectorizer()
train_replaced_na_keyword = train_df.copy()
train_replaced_na_keyword['keyword'] = train_df['keyword'].fillna(' ')
 train_vectors = count_vectorizer.fit_transform(train_replaced_na_keyword["keyword"])
 dev_replaced_na_keyword = dev_df.copy()
dev_replaced_na_keyword['keyword'] = dev_df['keyword'].fillna(' ')
 dev_vectors = count_vectorizer.transform(dev_replaced_na_keyword["keyword"])
 train_replaced_na_keyword
from stability_selection import plot_stability_path<filter>
train_vectors[1].todense()
ss_indices = np.where(ss.get_support() == True)[0]
ss_indices<import_modules>
clf = linear_model.RidgeClassifier()
clf.fit(train_vectors, train_df["target"])
 predictions = clf.predict(dev_vectors);
print(classification_report(dev_df['target'], predictions))
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import RepeatedStratifiedKFold
from mlxtend.feature_selection import SequentialFeatureSelector<define_search_model>
count_vectorizer = feature_extraction.text.CountVectorizer()
train_textandkeyword_df = train_df.copy()
train_textandkeyword_df['keyword'] = train_df['keyword'].fillna(' ')
 train_textandkeyword_df['textandkeyword'] = train_textandkeyword_df['text'] + " // " + train_textandkeyword_df['keyword']
train_vectors = count_vectorizer.fit_transform(train_textandkeyword_df["textandkeyword"])
 dev_textandkeyword_df = dev_df.copy()
dev_textandkeyword_df['keyword'] = dev_df['keyword'].fillna(' ')
 dev_textandkeyword_df['textandkeyword'] = dev_textandkeyword_df['text'] + " // " + dev_textandkeyword_df['keyword']
dev_vectors = count_vectorizer.transform(dev_textandkeyword_df["textandkeyword"])
pipe = Pipeline(steps=[ 
('scaler', StandardScaler()), 
('clf', LogisticRegression(solver='lbfgs', verbose=1, n_jobs=1, random_state=42))
])
 cv = RepeatedStratifiedKFold(2, 50, random_state=42)
 sfs = SequentialFeatureSelector(pipe, k_features=(5,10), floating=True,
scoring='roc_auc', cv=cv, verbose=1, n_jobs=-1 )<train_model>
clf = linear_model.RidgeClassifier()
clf.fit(train_vectors, train_df["target"])
 predictions = clf.predict(dev_vectors);
print(classification_report(dev_df['target'], predictions))
%%time
sfs.fit(X_train, y_train )<import_modules>
clf = linear_model.RidgeClassifier()
train_vectors = count_vectorizer.fit_transform(train_textandkeyword_df["text"])
 dev_vectors = count_vectorizer.transform(dev_textandkeyword_df["text"])
 clf.fit(train_vectors, train_df["target"])
 predictions = clf.predict(dev_vectors);
print(classification_report(dev_df['target'], predictions))
import matplotlib.pyplot as plt<import_modules>
vectorizer = TfidfVectorizer(stop_words='english')
 clf = linear_model.RidgeClassifier()
train_vectors = vectorizer.fit_transform(train_df["text"])
 dev_vectors = vectorizer.transform(dev_df["text"])
 clf.fit(train_vectors, train_df["target"])
 predictions = clf.predict(dev_vectors);
print(classification_report(dev_df['target'], predictions))
from sklearn.model_selection import GridSearchCV<import_modules>
vectorizer = TfidfVectorizer()
clf = linear_model.RidgeClassifier()
train_vectors = vectorizer.fit_transform(train_df["text"])
 dev_vectors = vectorizer.transform(dev_df["text"])
 clf.fit(train_vectors, train_df["target"])
 predictions = clf.predict(dev_vectors);
print(classification_report(dev_df['target'], predictions))
from sklearn.model_selection import GridSearchCV<choose_model_class>
clf = linear_model.RidgeClassifier()
pipe = Pipeline([('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
Cs = np.logspace(-5, 5, 21)
 params = {
'clf__C': Cs
}
pipe = Pipeline(steps=[ 
('scaler', StandardScaler()), 
('clf', LogisticRegression(solver='lbfgs', verbose=1, n_jobs=1, random_state=42))
])
 cv = RepeatedStratifiedKFold(2, 100, random_state=42)
 grid = GridSearchCV(estimator=pipe, param_grid=params, cv=cv,
return_train_score=True,
scoring='roc_auc', verbose=1, n_jobs=-1 )<train_model>
class ClfSwitcher(BaseEstimator):
def __init__(self, estimator = linear_model.RidgeClassifier()):

 self.estimator = estimator
def fit(self, X, y=None, **kwargs):
self.estimator.fit(X, y)
 return self
def predict(self, X, y=None):
return self.estimator.predict(X)
 def predict_proba(self, X):
return self.estimator.predict_proba(X)
 def score(self, X, y):
return self.estimator.score(X, y )
%%time
grid.fit(X_train, y_train )<find_best_params>
%%time
clf = linear_model.RidgeClassifier()
pipeline = Pipeline([('tfidf', TfidfVectorizer(stop_words='english')) ,('clf', ClfSwitcher())])
 parameters = [
{
'clf__estimator': [SVC() ],
},
{
'clf__estimator': [SGDClassifier() ],
},
{
'clf__estimator': [MultinomialNB() ],
},
{
'clf__estimator': [linear_model.RidgeClassifier() ],
},
{
'clf__estimator': [MLPClassifier(random_state=1, max_iter=200, early_stopping=True)],
},
{
'clf__estimator': [RandomForestClassifier() ]
}
]
os.write(1, b"Starting grid search of models
")
 gscv = GridSearchCV(pipeline, parameters, cv=3, n_jobs=12, return_train_score=True, verbose=3, scoring='f1')
 gscv.fit(train_df["text"], train_df["target"] )
pipe = grid.best_estimator_
pipe<import_modules>
df = pd.DataFrame(gscv.cv_results_)
 df
import matplotlib.pyplot as plt<normalization>
%%time
clf = MultinomialNB()
pipe = Pipeline([('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
X_test = df_test.values
for t in [ss, sfs]:
X_test = t.transform(X_test )<train_model>
%%time
clf = MLPClassifier(random_state=1, max_iter=200, early_stopping=True)
 pipe = Pipeline([('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
pipe.fit(X_train, y_train )<predict_on_test>
pipeline = Pipeline([('tfidf', TfidfVectorizer()),('clf', MultinomialNB())])
 parameters = [
{
'clf__alpha': [1, 0, 0.5],
'clf__fit_prior': [True, False]
},
]
os.write(1, b"Starting grid search of alpha & fit_prior
")
 gscv = GridSearchCV(pipeline, parameters, cv=3, n_jobs=12, return_train_score=True, verbose=3, scoring='f1')
 gscv.fit(train_df["text"], train_df["target"] )
y_pred = pipe.predict_proba(X_test)[:, 1]<load_from_csv>
df = pd.DataFrame(gscv.cv_results_)
 df
df_subm = pd.read_csv('.. /input/sample_submission.csv', index_col='id' )<prepare_output>
clf = MultinomialNB(fit_prior=False)
 pipe = Pipeline([('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
df_subm[target] = y_pred<save_to_csv>
%%time
!pip install gensim
fastText_model = FastText.load('/kaggle/input/english-wikipedia-articles-20170820-models/enwiki_2017_08_20_fasttext.model')
 def randvec(w, n=50, lower=-1.0, upper=1.0):

 return np.array([random.uniform(lower, upper)for i in range(n)])
 def get_oov_fasttext(w):
twograms = ngrams(w, 2)
 vectors = []
for gram in twograms:
word_2gram = gram[0] + gram[1]
if word_2gram in fastText_model:
vectors.append(fastText_model[word_2gram])
 if(len(vectors)> 0):
return np.sum(vectors, axis=0)
 else:
return randvec(w, n=300)
 def fasttext_vec(w):

 if(w in fastText_model):
return fastText_model[w]
else:
return get_oov_fasttext(w)
 class FastTextTransformer(BaseEstimator, TransformerMixin):
def __init__(self, combine_strategy="concatenate", max_sentence_length=30):
assert(combine_strategy=="concatenate" or combine_strategy=='mean')
 self.combine_strategy = combine_strategy
self.max_sentence_length = max_sentence_length
self.empty_word_token = "EOF"
def transform(self, text_list):
texts = text_list.tolist()
result = [];
for text in texts:
vectors = [];
words = text.split()
if(self.combine_strategy == 'concatenate'):
max_index = self.max_sentence_length
else:
max_index = len(words)
 for index in range(max_index):
if(len(words)> index):
word = words[index]
else:
word = self.empty_word_token
vectors.append(fasttext_vec(word))
if(self.combine_strategy == 'concatenate'):
result.append(np.concatenate(vectors))
elif(self.combine_strategy == 'mean'):
result.append(np.mean(vectors, axis=0))
return result;
def fit(self, X, y=None):

 return self
df_subm.to_csv('submission.csv' )<import_modules>
os.write(1, b"Starting fasttext experiments
")
 clf = SGDClassifier()
pipe = Pipeline([('vectorizer', FastTextTransformer(combine_strategy='concatenate')) ,('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os
from sklearn import metrics
import sklearn.preprocessing, sklearn.kernel_ridge, sklearn.model_selection, sklearn.linear_model
import multiprocessing
import seaborn as sns
import scipy.stats 
<load_from_csv>
clf = SGDClassifier()
pipe = Pipeline([('vectorizer', FastTextTransformer(combine_strategy='mean')) ,('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
submission = pd.read_csv('.. /input/sample_submission.csv')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(f'Shape of the train {train.shape}')
 print(f'Shape of the test {test.shape}')
 test.head() 
<prepare_x_and_y>
clf = SGDClassifier()
pipe = Pipeline([('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
data = train.iloc[:, 2:].values
ref = train['target'].values
sns.countplot(train['target'])
<save_to_csv>
for dependency in("brown", "names", "wordnet", "averaged_perceptron_tagger", "universal_tagset"):
nltk.download(dependency )
model_default = sklearn.linear_model.LogisticRegression()
model_default.fit(data, ref)
 predict_test = model_default.predict_proba(test.iloc[:, 1:].values)
 submission['target'] = predict_test
submission.to_csv('submission_logreg_default.csv', index=False)
 model = sklearn.linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 model.fit(data, ref)
 predict_test = model.predict_proba(test.iloc[:, 1:].values)
 submission['target'] = predict_test
submission.to_csv('submission_logreg_l1.csv', index=False )<train_on_grid>
nlp = spacy.load("en", disable=['parser', 'tagger', 'ner'])
 nltk.download('stopwords')
 stops = stopwords.words("english")
 def remove_accented_chars(text):

 text = unidecode.unidecode(text)
 return text
def expand_contractions(text):

 return contractions.fix(text);
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',str(text))
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_punctuation(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 def text_normalizer(comment, lemmatize, lowercase, remove_stopwords, remove_accents, normalize_contractions, normalize_URL, normalize_emoji, normalize_html, normalize_punctuation):
if lowercase:
comment = comment.lower()
if(remove_accents):
comment = remove_accented_chars(comment)
 if(normalize_contractions):
comment = expand_contractions(comment)
 if(normalize_URL):
comment = remove_URL(comment)
 if(normalize_emoji):
comment = remove_emoji(comment)
 if(normalize_html):
comment = remove_html(comment)
 if(normalize_punctuation):
comment = remove_punctuation(comment)
 if(remove_stopwords):
comment = nlp(comment)
 words = [];
for token in comment:
if not remove_stopwords or(remove_stopwords and token.text not in stops):
words.append(token.text)
 comment = " ".join(words)
 if(lemmatize):
comment = nlp(comment)
 comment = " ".join(word.lemma_.strip() for word in comment)
 return comment
class PrePropTextTransformer(BaseEstimator, TransformerMixin):
def __init__(self, lemmatize=False, lowercase=False, remove_stopwords=False, remove_accents=False, normalize_contractions=False, normalize_URL=False, normalize_emoji=False, normalize_html=False, normalize_punctuation=False):
self.lemmatize=lemmatize
self.lowercase=lowercase
self.remove_stopwords=remove_stopwords
self.remove_accents=remove_accents
self.normalize_contractions=normalize_contractions
self.normalize_URL=normalize_URL
self.normalize_emoji=normalize_emoji
self.normalize_html=normalize_html
self.normalize_punctuation=normalize_punctuation
def transform(self, text_list):
texts = text_list.tolist()
result = [];
for text in texts:
result.append(text_normalizer(text, self.lemmatize, self.lowercase, self.remove_stopwords, self.remove_accents, self.normalize_contractions, self.normalize_URL, self.normalize_emoji, self.normalize_html, self.normalize_punctuation))
return pd.Series(result)
 def fit(self, X, y=None):

 return self
def cross_validation(X, y, model, parameters, pname, nfold=10):
cv_method = sklearn.model_selection.KFold(n_splits=nfold, shuffle=True, random_state=13)
 rgr = sklearn.model_selection.GridSearchCV(model, parameters, n_jobs=multiprocessing.cpu_count() -1, cv=cv_method, scoring='roc_auc')
 rgr.fit(X, y)
 plt.semilogx(parameters[pname], rgr.cv_results_['mean_test_score'], 'o-r')
 plt.xlabel(pname)
 plt.ylabel('ROC-AUC')
 plt.title(f'{nfold}-Fold cross validation')
 print(f"The best {pname} parameter is {rgr.best_params_[pname]}")
 return rgr 
<save_to_csv>
%%time
clf = MultinomialNB(fit_prior=False)
 pipe = Pipeline([('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
predict_test = rgr.best_estimator_.predict_proba(test.iloc[:, 1:].values)
 submission['target'] = predict_test
submission.to_csv('submission_logreg_cv_l1.csv', index=False )<find_best_model_class>
%%time
os.write(1, b"Starting experimetns w text preprop
")
 clf = MultinomialNB()
pipe = Pipeline([('preprop', PrePropTextTransformer()),('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
def cross_validation_average(X, y, X_test, model, nfold=10):
scores = []
y_test = 0
folds = sklearn.model_selection.StratifiedKFold(n_splits=nfold, shuffle=True, random_state=13)
 for idx,(train_index, valid_index)in enumerate(folds.split(X, y)) :
X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
print(f'Process fold no {idx}: train/validation={len(y_train)}/{len(y_valid)}')
 model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = metrics.roc_auc_score(y_valid, y_pred_valid)
 y_test += model.predict_proba(X_test)[:, 1]
scores.append(score)
 plt.plot(range(nfold), scores, 'o-r')
 plt.xlabel('N-Fold')
 plt.ylabel('ROC-AUC')
 plt.title(f'{nfold}-Fold cross validation scores')
 print(f"Results: Max={np.max(scores)}, Mean={np.mean(scores)}, STD={np.std(scores)}")
 return scores, y_test/nfold 
<save_to_csv>
%%time
clf = MultinomialNB(fit_prior=False)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=True,
lowercase=True,
remove_stopwords=True,
remove_accents=True,
normalize_contractions=True,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=True 
)) ,('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
scores, predict_test = cross_validation_average(data, ref, test.iloc[:, 1:].values, rgr.best_estimator_, nfold=10)
 submission['target'] = predict_test
submission.to_csv('submission_logreg_cv_l1_average.csv', index=False)
<load_from_csv>
%%time
pipeline = Pipeline([('preprop', PrePropTextTransformer()), 
('vectorizer', TfidfVectorizer()), 
('predictor', MultinomialNB(fit_prior=False)) ])
 parameters = [
{'preprop__lemmatize': [True, False]},
{'preprop__lowercase': [True, False]},
{'preprop__remove_stopwords': [True, False]},
{'preprop__remove_accents': [True, False]},
{'preprop__normalize_contractions': [True, False]},
{'preprop__normalize_URL': [True, False]},
{'preprop__normalize_emoji': [True, False]},
{'preprop__normalize_html': [True, False]},
{'preprop__normalize_punctuation': [True, False]}
]
try:
grid_search_pd = pd.read_csv("/kaggle/input/precomputedgridsearches2/text_preprop_gs_results.csv")
 except:
gscv = GridSearchCV(pipeline, parameters, cv=3, n_jobs=1, return_train_score=True, verbose=3, scoring='f1')
 gscv.fit(train_df["text"], train_df["target"])
 grid_search_pd = pd.DataFrame(gscv.cv_results_);
grid_search_pd.to_csv("/kaggle/input/precomputedgridsearches/text_preprop_gs_results.csv")
 grid_search_pd
submission = pd.read_csv('.. /input/sample_submission.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 train_df = pd.read_csv(".. /input/train.csv" )<set_options>
%%time
clf = MultinomialNB(fit_prior=False)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
%matplotlib inline<define_variables>
%%time
clf = MultinomialNB(fit_prior=False)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', TfidfVectorizer(tokenizer=TweetTokenizer().tokenize)) ,('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
print('
print('
print('% target = 1: {}%'.format(round(float(len(train_df[train_df['target'] == 1])) / len(train_df)* 100)))
 tax=round(float(len(train_df[train_df['target'] == 1])) / len(train_df)* 100 )<feature_engineering>
!pip install transformers
def mean_across_all_tokens(hidden_states):
return torch.mean(hidden_states[-1], dim=1)
 def sum_all_tokens(hidden_states):
return torch.sum(hidden_states[-1], dim=1)
 def concat_all_tokens(hidden_states):
batch_size, max_tokens, emb_dim = hidden_states[-1].shape
return torch.reshape(hidden_states[-1],(batch_size, max_tokens * emb_dim))
def CLS_token_embedding(hidden_states):
return hidden_states[-1][:, 0, :]
class BertTransformer(BaseEstimator, TransformerMixin):
def __init__(
 self,
max_length: int = 60,
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased"),
model = BertModel.from_pretrained("bert-base-uncased", output_hidden_states=True),
embedding_func = mean_across_all_tokens,
combine_sentence_tokens=True 
):
self.tokenizer = tokenizer;
self.combine_sentence_tokens = combine_sentence_tokens;
self.embedding_func = embedding_func;
self.model = model
self.model.eval()
self.max_length = max_length
def _tokenize(self, text_list: List[str])-> Tuple[torch.tensor, torch.tensor]:
input_ids = self.tokenizer.batch_encode_plus(text_list,
add_special_tokens=True,
max_length=self.max_length,
pad_to_max_length=True 
)["input_ids"]
return torch.LongTensor(input_ids)
 def _tokenize_and_predict(self, text_list: List[str])-> torch.tensor:
input_ids_tensor = self._tokenize(text_list)
 out = self.model(input_ids=input_ids_tensor)
 hidden_states = out[2]
if(self.combine_sentence_tokens):
return self.embedding_func(hidden_states)
 else:
return hidden_states[-1]
def transform(self, text_list: List[str], batch_size=32):
if isinstance(text_list, pd.Series):
text_list = text_list.tolist()
batches = minibatch(text_list, size=batch_size)
 predictions = []
for batch in batches:
with torch.no_grad() :
batch_predictions = self._tokenize_and_predict(batch)
 predictions.append(batch_predictions)
 return torch.cat(predictions, dim=0)
 def fit(self, X, y=None):

 return self
for df in [test_df, train_df]:
d=pd.Series(d)
 d=(d + 1 ).tolist()
for x in d:
df[df.columns[x]]=(df[df.columns[x]]-df[df.columns[x]].min())/(df[df.columns[x]].max() -df[df.columns[x]].min())
 df[df.columns[x]]=np.log(( 1+df[df.columns[x]]))<define_variables>
bertTransformer = BertTransformer(combine_sentence_tokens=False)
 bertTransformer.transform(["pablo", "I love Pablo"] ).shape
sample= ['target','16', '33', '80', '91', '217', '295']<feature_engineering>
%%time
os.write(1, b"Starting experiments with BERT
")
 clf = linear_model.RidgeClassifier()
pipe = Pipeline([('vectorizer', BertTransformer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
<create_dataframe>
%%time
clf = linear_model.RidgeClassifier()
pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
result=pd.DataFrame()<prepare_x_and_y>
%%time
clf = MLPClassifier(random_state=1, max_iter=200, early_stopping=True)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
y = train_df['target']
X = train_df.drop(['target','id'], axis=1)
 X_ol=X[d]
print('Train set:', X.shape, y.shape, X_ol.shape )<train_model>
%%time
clf = MLPClassifier(random_state=1, max_iter=100, early_stopping=True)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer()),('clf', clf)])
 parameters = [
{'clf__hidden_layer_sizes': [(100, 100),(100,),(100, 50),(50, 50)]}
]
try:
grid_search_pd = pd.read_csv("/kaggle/input/precomputedgridsearches2/mlp_clf_gs_results.csv")
 except:
gscv = GridSearchCV(pipe, parameters, cv=2, n_jobs=1, return_train_score=True, verbose=3, scoring='f1')
 gscv.fit(train_df["text"], train_df["target"])
 grid_search_pd = pd.DataFrame(gscv.cv_results_);
grid_search_pd.to_csv("/kaggle/input/precomputedgridsearches2/mlp_clf_gs_results.csv")
 grid_search_pd
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=6)
 print('Train set:', X_train.shape, y_train.shape)
 print('Test set:', X_test.shape, y_test.shape)
 X_train_ol=X_train[d]
X_test_ol=X_test[d]
print('Train set select:', X_train_ol.shape, y_train.shape)
 print('Test set select:', X_test_ol.shape, y_test.shape )<prepare_x_and_y>
%%time
clf = MLPClassifier(random_state=1, max_iter=100, early_stopping=True)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer()),('clf', clf)])
 parameters = [
{'clf__hidden_layer_sizes': [(100, 50, 50),(100,100, 50)]}
]
try:
grid_search_pd = pd.read_csv("/kaggle/input/precomputedgridsearches2/mlp_clf2_gs_results.csv")
 except:
gscv = GridSearchCV(pipe, parameters, cv=2, n_jobs=1, return_train_score=True, verbose=3, scoring='f1')
 gscv.fit(train_df["text"], train_df["target"])
 grid_search_pd = pd.DataFrame(gscv.cv_results_);
grid_search_pd.to_csv("/kaggle/input/precomputedgridsearches2/mlp_clf2_gs_results.csv")
 grid_search_pd
print('% target = 1 in train set: {}%'.format(round(float(len(y_train[y_train==1])) / len(y_train)* 100)))
 print('% target = 1 in test set: {}%'.format(round(float(len(y_test[y_test==1])) / len(y_test)* 100)) )<train_model>
bertTransformer = BertTransformer()
bertTransformer.transform(["granola bars"] ).shape
regr_all = linear_model.LinearRegression(fit_intercept=True, normalize=True)
 regr_all.fit(X_train,y_train )<compute_test_metric>
%%time
clf = MLPClassifier(random_state=1, max_iter=100, early_stopping=True)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer()),('predictor', clf)])
 parameters = [
{'vectorizer__embedding_func': [sum_all_tokens, mean_across_all_tokens, concat_all_tokens, CLS_token_embedding]}
]
try:
grid_search_pd = pd.read_csv("/kaggle/input/precomputedgridsearches2/emb_funcs_gs_results.csv")
 except:
gscv = GridSearchCV(pipe, parameters, cv=2, n_jobs=1, return_train_score=True, verbose=3, scoring='f1')
 gscv.fit(train_df["text"], train_df["target"])
 grid_search_pd = pd.DataFrame(gscv.cv_results_);
grid_search_pd.to_csv("/kaggle/input/precomputedgridsearches2/emb_funcs_gs_results.csv")
 grid_search_pd
y_hat= regr_all.predict(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat)[100-tax])
 result['regr_all']=y_hat
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat - y_test)** 2))
print('Variance score: %.2f' % regr_all.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat))<train_model>
%%time
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
 model = RobertaModel.from_pretrained('roberta-base', output_hidden_states=True)
 clf = XGBClassifier(n_estimators=300)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer(model=model, tokenizer=tokenizer)) ,('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
regr_ol = linear_model.LinearRegression(fit_intercept=True, normalize=True)
 regr_ol.fit(X_train_ol,y_train )<compute_test_metric>
%%time
os.write(1, b"Starting experiments with sequential models
")
 bertTransformer = BertTransformer(combine_sentence_tokens=False)
 X_train = bertTransformer.transform(train_df["text"])
 X_dev = bertTransformer.transform(dev_df["text"])
 y_train = train_df["target"].tolist()
y_dev = dev_df["target"].tolist()
regr_ol.predict(X_test_ol)
 y_hat=y_hat*0.5/(np.sort(y_hat)[100-tax])
 result['regr_ol']=y_hat
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat - y_test)** 2))
print('Variance score: %.2f' % regr_ol.score(X_test_ol, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat))<sort_values>
%%time
torch_rnn = TorchRNNClassifier(
 vocab=[],
use_embedding=False,
bidirectional=False,
hidden_dim=50,
max_iter=50,
eta=0.05)
 _ = torch_rnn.fit(X_train, y_train )
best=score[score['AUC ROC']>score['AUC ROC'].max() *0.95]
best<train_model>
%%time
predictions = torch_rnn.predict(X_dev)
 print(classification_report(y_dev, predictions))
regr_sel = RFE(regr_all, 4, step=1,verbose=0)
 regr_sel.fit(X_train,y_train )<compute_test_metric>
%%time
torch_bi_lstm = TorchRNNClassifier(
 vocab=[],
use_embedding=False,
bidirectional=True,
hidden_dim=50,
max_iter=50,
eta=0.05)
 _ = torch_bi_lstm.fit(X_train, y_train )
y_hat= regr_sel.predict(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat)[100-tax])
 result['regre_sel']=y_hat
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat - y_test)** 2))
print('Variance score: %.2f' % regr_sel.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat))<choose_model_class>
%%time
predictions = torch_bi_lstm.predict(X_dev)
 print(classification_report(y_dev, predictions))
param_grid = { 'criterion': ['gini','entropy'],
'splitter': ['best','random'],
'max_depth': [2,4,10,20,None],
'max_features' : ['auto', 'sqrt', 'log2']}
Tree = DecisionTreeClassifier()<train_on_grid>
%%time
torch_bilstm_attn_manning2015 = TorchRNNClassifier(
 vocab=[],
use_embedding=False,
attention="GlobalAttnManning2015",
bidirectional=True,
hidden_dim=50,
max_iter=30,
eta=0.05)
 _ = torch_bilstm_attn_manning2015.fit(X_train, y_train )
grid = GridSearchCV(estimator=Tree, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_<train_model>
%%time
predictions = torch_bilstm_attn_manning2015.predict(X_dev)
 print(classification_report(y_dev, predictions))
Tree = DecisionTreeClassifier(**best_param)
 Tree.fit(X_train,y_train )<compute_test_metric>
%%time
torch_bilstm_attn_ShouPeng2016 = TorchRNNClassifier(
 vocab=[],
use_embedding=False,
attention="AttnShouPeng2016",
bidirectional=True,
hidden_dim=50,
max_iter=30,
eta=0.05)
 _ = torch_bilstm_attn_ShouPeng2016.fit(X_train, y_train )
y_hat= Tree.predict(X_test)
 result['Tree']=y_hat
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat - y_test)** 2))
print('Variance score: %.2f' % Tree.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat))<train_on_grid>
%%time
predictions = torch_bilstm_attn_ShouPeng2016.predict(X_dev)
 print(classification_report(y_dev, predictions))
grid = GridSearchCV(estimator=Tree, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train_ol,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_<train_model>
class BertClassifierPredictor(BaseEstimator, ClassifierMixin):
def __init__(self, model=ClassificationModel('roberta', 'roberta-base', args={"overwrite_output_dir": True},
use_cuda=False)) :
self.model = model
logging.basicConfig(level=logging.INFO)
 transformers_logger = logging.getLogger("transformers")
 transformers_logger.setLevel(logging.WARNING)
 def fit(self, X, y):
X = X.tolist()
y = y.tolist()
self.classes_ = unique_labels(y)
 self.X_ = X
self.y_ = y
d = {'text': X, 'labels': y}
df = pd.DataFrame(data=d)
 self.model.train_model(df)
 return self
def predict(self, X):
return self.model.predict(X.tolist())[0]
Tree_ol = DecisionTreeClassifier(**best_param)
 Tree_ol.fit(X_train_ol,y_train )<compute_test_metric>
%%time
os.write(1, b"Starting experiments w BERT finetuning
")
 model = ClassificationModel('bert', '/kaggle/input/simpletransformer-outputs/checkpoint-bert-epoch-1', args={"overwrite_output_dir": True}, use_cuda=False)
 clf = BertClassifierPredictor(model=model)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('predictor', clf)])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
y_hat= Tree_ol.predict(X_test_ol)
 result['Tree_ol']=y_hat
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat - y_test)** 2))
print('Variance score: %.2f' % Tree_ol.score(X_test_ol, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat))<train_model>
%%time
logging.basicConfig(level=logging.INFO)
 transformers_logger = logging.getLogger("transformers")
 transformers_logger.setLevel(logging.WARNING)
 model = ClassificationModel('roberta', '/kaggle/input/simpletransformer-outputs/checkpoint-roberta-epoch-1', args={"overwrite_output_dir": True}, use_cuda=False)
 clf = BertClassifierPredictor(model=model)
 pipe_roberta_finetuned = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('predictor', clf)])
 predictions = pipe_roberta_finetuned.predict(dev_df['text'])
 print(classification_report(dev_df['target'], predictions))
Tree_selec = RFE(Tree, 94, step=1,verbose=0)
 Tree_selec.fit(X_train,y_train )<compute_test_metric>
%%time
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
 model = RobertaModel.from_pretrained('roberta-base', output_hidden_states=True)
 clf = MLPClassifier(random_state=1, max_iter=200, early_stopping=True)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', BertTransformer(model=model, tokenizer=tokenizer)) ,('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
y_hat= Tree_selec.predict(X_test)
 result['Tree_selec']=y_hat
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat - y_test)** 2))
print('Variance score: %.2f' % Tree_selec.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat))<choose_model_class>
%%time
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
 model = RobertaModel.from_pretrained('roberta-base', output_hidden_states=True)
 bertTransformer = BertTransformer(model=model, tokenizer=tokenizer, combine_sentence_tokens=False)
 X_train = bertTransformer.transform(train_df["text"])
 X_dev = bertTransformer.transform(dev_df["text"])
 y_train = train_df["target"].tolist()
y_dev = dev_df["target"].tolist()
param_grid = { 'C': [0.001,0.1,10.0,100.0],
'fit_intercept': [True,False],
'class_weight': ['balanced', None],
'solver' : ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']}
LR = LogisticRegression()<train_on_grid>
%%time
torch_bilstm_attention_roberta_exp = TorchRNNClassifier(
 vocab=[],
use_embedding=False,
attention="AttnShouPeng2016",
bidirectional=True,
hidden_dim=50,
max_iter=30,
eta=0.05)
 _ = torch_bilstm_attention_roberta_exp.fit(X_train, y_train )
grid = GridSearchCV(estimator=LR, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_<train_model>
%%time
predictions = torch_bilstm_attention_roberta_exp.predict(X_dev)
 print(classification_report(y_dev, predictions))
LR = LogisticRegression(**best_param)
 LR.fit(X_train,y_train )<compute_test_metric>
%%time
os.write(1, b"Starting experiments with spacyclassifier
")
 spacy_classifier = SpacyClassifier(n_iter=50)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('predictor', spacy_classifier)])
 pipe.fit(train_df["text"], train_df["target"] )
y_hat= LR.predict_proba(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['LR']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % LR.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))<train_on_grid>
%%time
predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
grid = GridSearchCV(estimator=LR, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train_ol,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_<train_model>
%%time
checkpoint_path = "/kaggle/input/models3/GPT2_CLS_CHECKPOINT_ephoc20.pth.tar"
gpt2_classifier = GPT2Classifier(max_iter=20, finetune_GPT2=False, batch_size=32, checkpoint_path=checkpoint_path,
base_dir=".", classes=[0,1])
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('predictor', gpt2_classifier)])
 pipe.fit(train_df["text"], train_df["target"])
LR_ol = LogisticRegression(**best_param)
 LR_ol.fit(X_train_ol,y_train )<compute_test_metric>
%%time
predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions))
y_hat= LR_ol.predict_proba(X_test_ol)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['LR_ol']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % LR_ol.score(X_test_ol, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))<sort_values>
entire_train_df = pd.concat([train_df, dev_df] )
best=score[score['AUC ROC']>score['AUC ROC'].max() *0.95]
best<train_model>
%%time
model = ClassificationModel('roberta', '/kaggle/input/simpletransformer-outputs/checkpoint-roberta_submission-epoch-2', args={"overwrite_output_dir": True, "num_train_epochs": 2}, use_cuda=False)
 clf = BertClassifierPredictor(model=model)
 pipe_roberta_finetuned_submit = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('predictor', clf)])
LR_selec = RFE(LR, 14, step=1,verbose=0)
 LR_selec.fit(X_train,y_train )<compute_test_metric>
test_vectors = test_df['text']
sample_submission = pd.read_csv("/kaggle/input/data-baby2/sample_submission.csv")
 sample_submission["target"] = pipe_roberta_finetuned_submit.predict(test_vectors)
 sample_submission.head()
<choose_model_class><EOS> 
sample_submission.to_csv("submission.csv", index=False )