Kaggle competition: COVID19 Global Forecasting (Week 1)
import sys, os, re, csv, codecs, numpy as np, pandas as pd
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, GRU, Embedding, Dropout, Activation
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
from nltk.tokenize import TweetTokenizer
from unidecode import unidecode
import gc
from keras import backend as K<import_modules>
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv')
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/submission.csv' )
print(tf.__version__)
 tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None
)<define_variables>
pio.templates.default = "plotly_dark"
py.init_notebook_mode(connected=True )
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE1=f'{path}glove6b50d/glove.6B.50d.txt'
TRAIN_DATA_FILE=f'{path}{comp}train.csv'
TEST_DATA_FILE=f'{path}{comp}test.csv'
EMBEDDING_FILE2=f'{path}fasttext-crawl-300d-2m/rawl-300d-2M.vec'
categories = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]<load_from_csv>
latest_grouped = train.groupby('Country/Region')['ConfirmedCases', 'Fatalities'].sum().reset_index()
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv" )<define_variables>
europe = list(['Austria','Belgium','Bulgaria','Croatia','Cyprus','Czechia','Denmark','Estonia','Finland','France','Germany','Greece','Hungary','Ireland',
'Italy', 'Latvia','Luxembourg','Lithuania','Malta','Norway','Netherlands','Poland','Portugal','Romania','Slovakia','Slovenia',
'Spain', 'Sweden', 'United Kingdom', 'Iceland', 'Russia', 'Switzerland', 'Serbia', 'Ukraine', 'Belarus',
'Albania', 'Bosnia and Herzegovina', 'Kosovo', 'Moldova', 'Montenegro', 'North Macedonia'])
 europe_grouped_latest = latest_grouped[latest_grouped['Country/Region'].isin(europe)]
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220<define_variables>
train['province_encoded'] = train['Province/State'].apply(lambda x: province_encoded[x])
 train.head()
embedding_dim_facebook = 300
embedding_dim_google = 300
max_features = 20000
maxlen = 100<define_variables>
train['country_encoded'] = train['Country/Region'].apply(lambda x: country_encoded[x])
 train.head()
training_samples_count = 149571
validation_samples_count = 10000
length_threshold = 20000
word_count_threshold = 900
words_limit = 310000
valid_characters = " " + "@$" + "'!?-" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz".upper()
valid_characters_ext = valid_characters + "abcdefghijklmnopqrstuvwxyz".upper()
valid_set = set(x for x in valid_characters)
 valid_set_ext = set(x for x in valid_characters_ext)
 toxic_words = ["poop", "crap", "prick", "twat", "wikipedia", "wiki", "hahahahaha", "lol", "bastard", "sluts", "slut", "douchebag", "douche", "blowjob", "nigga", "dumb", "jerk", "wanker", "wank", "penis", "motherfucker", "fucker", "fuk", "fucking", "fucked", "fuck", "bullshit", "shit", "stupid", "bitches", "bitch", "suck", "cunt", "dick", "cocks", "cock", "die", "kill", "gay", "jewish", "jews", "jew", "niggers", "nigger", "faggot", "fag", "asshole"]
astericks_words = [('mother****ers', 'motherfuckers'),('motherf*cking', 'motherfucking'),('mother****er', 'motherfucker'),('motherf*cker', 'motherfucker'),('bullsh*t', 'bullshit'),('f**cking', 'fucking'),('f*ucking', 'fucking'),('fu*cking', 'fucking'),('****ing', 'fucking'),('a**hole', 'asshole'),('assh*le', 'asshole'),('f******', 'fucking'),('f*****g', 'fucking'),('f***ing', 'fucking'),('f**king', 'fucking'),('f*cking', 'fucking'),('fu**ing', 'fucking'),('fu*king', 'fucking'),('fuc*ers', 'fuckers'),('f*****', 'fucking'),('f***ed', 'fucked'),('f**ker', 'fucker'),('f*cked', 'fucked'),('f*cker', 'fucker'),('f*ckin', 'fucking'),('fu*ker', 'fucker'),('fuc**n', 'fucking'),('ni**as', 'niggas'),('b**ch', 'bitch'),('b*tch', 'bitch'),('c*unt', 'cunt'),('f**ks', 'fucks'),('f*ing', 'fucking'),('ni**a', 'nigga'),('c*ck', 'cock'),('c*nt', 'cunt'),('cr*p', 'crap'),('d*ck', 'dick'),('f***', 'fuck'),('f**k', 'fuck'),('f*ck', 'fuck'),('fc*k', 'fuck'),('fu**', 'fuck'),('fu*k', 'fuck'),('s***', 'shit'),('s**t', 'shit'),('sh**', 'shit'),('sh*t', 'shit'),('tw*t', 'twat')]
fasttext_misspelings = {"'n'balls": 'balls', "-nazi's": 'nazis', 'adminabuse': 'admin abuse', "admins's": 'admins', 'arsewipe': 'arse wipe', 'assfack': 'asshole', 'assholifity': 'asshole', 'assholivity': 'asshole', 'asshoul': 'asshole', 'asssholeee': 'asshole', 'belizeans': 'mexicans', "blowing's": 'blowing', 'bolivians': 'mexicans', 'celtofascists': 'fascists', 'censorshipmeisters': 'censor', 'chileans': 'mexicans', 'clerofascist': 'fascist', 'cowcrap': 'crap', 'crapity': 'crap', "d'idiots": 'idiots', 'deminazi': 'nazi', 'dftt': "don't feed the troll", 'dildohs': 'dildo', 'dramawhores': 'drama whores', 'edophiles': 'pedophiles', 'eurocommunist': 'communist', 'faggotkike': 'faggot', 'fantard': 'retard', 'fascismnazism': 'fascism', 'fascistisized': 'fascist', 'favremother': 'mother', 'fuxxxin': 'fucking', "g'damn": 'goddamn', 'harassmentat': 'harassment', 'harrasingme': 'harassing me', 'herfuc': 'motherfucker', 'hilterism': 'fascism', 'hitlerians': 'nazis', 'hitlerites': 'nazis', 'hubrises': 'pricks', 'idiotizing': 'idiotic', 'inadvandals': 'vandals', "jackass's": 'jackass', 'jiggabo': 'nigga', 'jizzballs': 'jizz balls', 'jmbass': 'dumbass', 'lejittament': 'legitimate', "m'igger": 'nigger', "m'iggers": 'niggers', 'motherfacking': 'motherfucker', 'motherfuckenkiwi': 'motherfucker', 'muthafuggas': 'niggas', 'nazisms': 'nazis', 'netsnipenigger': 'nigger', 'niggercock': 'nigger', 'niggerspic': 'nigger', 'nignog': 'nigga', 'niqqass': 'niggas', "non-nazi's": 'not a nazi', 'panamanians': 'mexicans', 'pedidiots': 'idiots', 'picohitlers': 'hitler', 'pidiots': 'idiots', 'poopia': 'poop', 'poopsies': 'poop', 'presumingly': 'obviously', 'propagandaanddisinformation': 'propaganda and disinformation', 'propagandaministerium': 'propaganda', 'puertoricans': 'mexicans', 'puertorricans': 'mexicans', 'pussiest': 'pussies', 'pussyitis': 'pussy', 'rayaridiculous': 'ridiculous', 'redfascists': 'fascists', 'retardzzzuuufff': 'retard', "revertin'im": 'reverting', 'scumstreona': 'scums', 'southamericans': 'mexicans', 'strasserism': 'fascism', 'stuptarded': 'retarded', "t'nonsense": 'nonsense', "threatt's": 'threat', 'titoists': 'communists', 'twatbags': 'douchebags', 'youbollocks': 'you bollocks'}
acronym_words = {} 
<define_variables>
from datetime import datetime
import time
cont_patterns = [ 
(r'(W|w)on't', r'will not'), 
(r'(C|c)an't', r'can not'), 
(r'(I|i)'m', r'i am'), 
(r'(A|a)in't', r'is not'), 
(r'(\w+)'ll', r'\g<1> will'), 
(r'(\w+)n't', r'\g<1> not'), 
(r'(\w+)'ve', r'\g<1> have'), 
(r'(\w+)'s', r'\g<1> is'), 
(r'(\w+)'re', r'\g<1> are'), 
(r'(\w+)'d', r'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def split_word(word, toxic_words):
if word == "":
return ""
lower = word.lower()
for toxic_word in toxic_words:
start = lower.find(toxic_word)
 if start >= 0:
end = start + len(toxic_word)
 result = " ".join([word[0:start], word[start:end], split_word(word[end:], toxic_words)])
 return result.replace(" ", " " ).strip()
return word
tknzr = TweetTokenizer(strip_handles=False, reduce_len=True)
 def word_tokenize(sentence):
sentence = sentence.replace("$", "s")
 sentence = sentence.replace("@", "a")
 sentence = sentence.replace("!", " ! ")
 sentence = sentence.replace("?", " ? ")
 return tknzr.tokenize(sentence)
 def replace_url(word):
if "http://" in word or "www." in word or "https://" in word or "wikipedia.org" in word:
return ""
return word
def normalize_by_dictionary(normalized_word, dictionary):
result = []
for word in normalized_word.split() :
if word == word.upper() :
if word.lower() in dictionary:
result.append(dictionary[word.lower() ].upper())
 else:
result.append(word)
 else:
if word.lower() in dictionary:
result.append(dictionary[word.lower() ])
 else:
result.append(word)
 return " ".join(result )<categorify>
nlp = spacy.load('en')
 def normalize_comment(comment):
comment = unidecode(comment)
 comment = comment[:length_threshold]
normalized_words = []
for w in astericks_words:
if w[0] in comment:
comment = comment.replace(w[0], w[1])
 if w[0].upper() in comment:
comment = comment.replace(w[0].upper() , w[1].upper())
 for word in word_tokenize(comment):
if word == "." or word == ",":
normalized_words.append(word)
 continue
word = replace_url(word)
 if word.count(".")== 1:
word = word.replace(".", " ")
 filtered_word = "".join([x for x in word if x in valid_set])
 normalized_word = split_word(filtered_word, toxic_words)
 normalized_word = normalize_by_dictionary(normalized_word, fasttext_misspelings)
 normalized_word = normalize_by_dictionary(normalized_word, acronym_words)
 normalized_words.append(normalized_word)
 normalized_comment = " ".join(normalized_words)
 result = []
for word in normalized_comment.split() :
if word.upper() == word:
result.append(word)
 else:
result.append(word.lower())
 result = " ".join(result)
 if "sock puppet" in result:
result = result.replace("sock puppet", "sockpuppet")
 if "SOCK PUPPET" in result:
result = result.replace("SOCK PUPPET", "SOCKPUPPET")
 return result<load_from_csv>
def read_data_files(train_filepath, test_filepath):
train = pd.read_csv(train_filepath)
 labels = train[categories].values
test = pd.read_csv(test_filepath)
 test_comments = test["comment_text"].fillna("_na_" ).values
np_normalize = np.vectorize(normalize_comment)
 comments = train["comment_text"].fillna("_na_" ).values
normalized_comments = np_normalize(comments)
 del comments
gc.collect()
comments = test["comment_text"].fillna("_na_" ).values
normalized_test_comments = np_normalize(test_comments)
 del comments
gc.collect()
print('Shape of data tensor:', normalized_comments.shape)
 print('Shape of label tensor:', labels.shape)
 print('Shape of test data tensor:', normalized_test_comments.shape)
 return(labels, normalized_comments, normalized_test_comments )<load_from_csv>
train['Mon'] = train['Date'].apply(lambda x: int(x.split('-')[1]))
train['Day'] = train['Date'].apply(lambda x: int(x.split('-')[2]))
%%time
labels, x_train, x_test = read_data_files(TRAIN_DATA_FILE, TEST_DATA_FILE )<string_transform>
train['serial'] = train['Mon'] * 30 + train['Day']
train.head()
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(x_train))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<load_pretrained>
train['serial'] = train['serial'] - train['serial'].min()
%%time
embedding_matrix = np.load('.. /input/embedding-2/embedding_matrix_big.npy' )<split>
gdp2020 = pd.read_csv('/kaggle/input/gdp2020/GDP2020.csv')
 population2020 = pd.read_csv('/kaggle/input/population2020/population2020.csv' )
x_train, x_valid, y_train, y_valid = train_test_split(x_train, labels, test_size = 0.1 )<choose_model_class>
gdp2020 = gdp2020.rename(columns={"rank":"rank_gdp"})
 gdp2020_numeric_list = [list(gdp2020)[0]] + list(gdp2020)[2:-1]
gdp2020.head()
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<train_model>
map_state = {'US':'United States',
'Korea, South':'South Korea',
'Cote d'Ivoire':'Ivory Coast',
'Czechia':'Czech Republic',
'Eswatini':'Swaziland',
'Holy See':'Vatican City',
'Jersey':'United Kingdom',
'North Macedonia':'Macedonia',
'Taiwan*':'Taiwan',
'occupied Palestinian territory':'Palestine'
}
map_state_rev = {v: k for k, v in map_state.items() }
%%time
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 x_train,
y_train,
validation_data =(x_valid, y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.5 ** global_epoch))
] 
)
 val_preds = model.predict(x_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(y_valid[:,i], val_preds[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
np.save('pred', pred)
 model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5' )<save_to_csv>
population2020['name'] = population2020['name'].apply(lambda x: map_state_rev[x] if x in map_state_rev else x)
 gdp2020['country'] = gdp2020['country'].apply(lambda x: map_state_rev[x] if x in map_state_rev else x )
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )<define_variables>
population2020 = population2020.rename(columns={"rank":"rank_pop"})
 population2020_numeric_list = [list(population2020)[0]] + list(gdp2020)[2:]
population2020.head()
f_caps_gru = '.. /input/capsule-net-with-gru/submission.csv'
f_dual_embed_pl = '.. /input/submission-dual-embed-pl/submission_dual_embed(2 ).csv'
f_dual_embed_mish = '.. /input/bi-gru-lstm-dual-embedding-with-mish/submission.csv'
f_lstm_glove_tta = '.. /input/improved-lstm-baseline-glove-dropout-trainta/submission.csv'
f_lstm_fast = '.. /input/improved-lstm-baseline-fasttext-dropout/submission.csv'
f_nbsvm = '.. /input/nb-svm-strong-linear-baseline/submission.csv'
f_rcn = '.. /input/rcn-capsule/10fold_capsule_am.csv'<load_from_csv>
train = pd.merge(train, population2020, how='left', left_on = 'Country/Region', right_on = 'name')
 train = pd.merge(train, gdp2020, how='left', left_on = 'Country/Region', right_on = 'country' )
p_caps_gru = pd.read_csv(f_caps_gru)
 p_dual_embed_pl = pd.read_csv(f_dual_embed_pl)
 p_dual_embed_mish = pd.read_csv(f_dual_embed_mish)
 p_lstm_glove_tta = pd.read_csv(f_lstm_glove_tta)
 p_lstm_fast = pd.read_csv(f_lstm_fast)
 p_nbsvm = pd.read_csv(f_nbsvm)
 p_rcn = pd.read_csv(f_rcn )<feature_engineering>
train.isnull().sum()
label_cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
p_res = p_caps_gru.copy()
p_res[label_cols] =(p_caps_gru[label_cols] + p_dual_embed_pl[label_cols] + p_dual_embed_mish[label_cols] + p_lstm_glove_tta[label_cols] + p_lstm_fast[label_cols] + p_nbsvm[label_cols] + p_rcn[label_cols] * 3)/ 9<save_to_csv>
train.isnull().sum()
p_res.to_csv('submission.csv', index=False )<set_options>
train = train.fillna(0 )
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
numeric_features_X = ['Lat','Long', 'province_encoded' ,'country_encoded','Mon','Day'] + population2020_numeric_list + gdp2020_numeric_list
numeric_features_Y = ['ConfirmedCases', 'Fatalities']
train_numeric_X = train[numeric_features_X]
train_numeric_Y = train[numeric_features_Y]
from fastai.text import *
from fastai import *<load_from_csv>
test['province_encoded'] = test['Province/State'].apply(lambda x: province_encoded[x] if x in province_encoded else max(province_encoded.values())+1 )
df_train = pd.read_csv(path/'train.csv')
 df_test = pd.read_csv(path/'test.csv')
 df = df_train.append(df_test)
 df.head()<define_variables>
test['country_encoded'] = test['Country/Region'].apply(lambda x: country_encoded[x] if x in country_encoded else max(country_encoded.values())+1 )
data_lm.show_batch()<load_pretrained>
test['Mon'] = test['Date'].apply(lambda x: int(x.split('-')[1]))
test['Day'] = test['Date'].apply(lambda x: int(x.split('-')[2]))
learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3 )<train_model>
test['serial'] = test['Mon'] * 30 + test['Day']
test['serial'] = test['serial'] - test['serial'].min()
learn.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))<save_model>
test = pd.merge(test, population2020, how='left', left_on = 'Country/Region', right_on = 'name')
 test = pd.merge(test, gdp2020, how='left', left_on = 'Country/Region', right_on = 'country' )
learn.save('fit_head' )<train_model>
learn.load('fit_head');<train_model>
test_numeric_X = test[numeric_features_X]
test_numeric_X.isnull().sum()
learn.fit_one_cycle(6, 1e-3, moms=(0.8,0.7))<save_model>
test_numeric_X = test_numeric_X.fillna(-1 )
learn.save('fine_tuned' )<load_pretrained>
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
learn.load('fine_tuned');<define_variables>
pipeline = Pipeline([('scaler', StandardScaler()),('estimator', LinearRegression())])
 pipeline.fit(train_numeric_X, train_numeric_Y )
TEXT = "I liked this movie because"
N_WORDS = 40
N_SENTENCES = 2<predict_on_test>
predicted = pipeline.predict(test_numeric_X )
print("
".join(learn.predict(TEXT, N_WORDS, temperature=0.75)for _ in range(N_SENTENCES)) )<categorify>
from sklearn.svm import SVR
learn.save_encoder('fine_tuned_enc' )<load_from_csv>
pipeline = Pipeline([('scaler', StandardScaler()),('estimator', SVR())])
 pipeline.fit(train_numeric_X, train_numeric_Y.values[:,0])
 pipeline2 = Pipeline([('scaler', StandardScaler()),('estimator', SVR())])
 pipeline2.fit(train_numeric_X, train_numeric_Y.values[:,1])
 discovered, fatal = pipeline.predict(test_numeric_X), pipeline2.predict(test_numeric_X )
test = pd.read_csv(path/"test.csv")
 test_datalist = TextList.from_df(test, cols='comment_text')
<load_from_csv>
data_cls =(TextList.from_csv(path, 'train.csv', cols='comment_text', vocab=data_lm.vocab)
.split_by_rand_pct(valid_pct=0.1)
.label_from_df(cols=['toxic', 'severe_toxic','obscene', 'threat', 'insult', 'identity_hate'], label_cls=MultiCategoryList, one_hot=True)
.add_test(test_datalist)
.databunch())
 data_cls.save('data_clas.pkl')
<load_pretrained>
from sklearn.neighbors import KNeighborsClassifier
data_clas = load_data(path, 'data_clas.pkl', bs=bs )<define_variables>
from sklearn.neighbors import KNeighborsClassifier
data_clas.show_batch()<load_pretrained>
from sklearn.neighbors import KNeighborsClassifier
learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5)
 learn.load_encoder('fine_tuned_enc' )<train_model>
pipeline = Pipeline([('scaler', StandardScaler()),('estimator', KNeighborsClassifier(n_jobs=4)) ])
 pipeline.fit(train_numeric_X, train_numeric_Y )
learn.fit_one_cycle(1, 2e-2, moms=(0.8,0.7))<save_model>
from sklearn.ensemble import RandomForestClassifier
learn.save('first' )<load_pretrained>
RF_model = RandomForestClassifier(n_estimators=50,n_jobs=4,verbose=True)
 RF_model.fit(train_numeric_X, train_numeric_Y )
learn.load('first');<train_model>
learn.freeze_to(-2)
 learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))<save_model>
from sklearn.ensemble import AdaBoostClassifier
learn.save('second');
learn.load('second');<train_model>
adaboost_model_for_ConfirmedCases = AdaBoostClassifier(n_estimators=15)
 adaboost_model_for_ConfirmedCases.fit(train_numeric_X, train_numeric_Y[numeric_features_Y[0]])
 adaboost_model_for_Fatalities = AdaBoostClassifier(n_estimators=15)
 adaboost_model_for_Fatalities.fit(train_numeric_X, train_numeric_Y[numeric_features_Y[1]] )
learn.freeze_to(-3)
 learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))<save_model>
learn.save('third')
 learn.load('third');<train_model>
learn.unfreeze()
learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))<predict_on_test>
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn import model_selection
from mlxtend.classifier import StackingCVClassifier
learn.predict("I really loved that movie, it was awesome!" )<predict_on_test>
clf1 = KNeighborsClassifier(n_neighbors=100)
 clf2 = RandomForestClassifier(n_estimators=10)
 clf3 = GaussianNB()
lr = LogisticRegression(solver='lbfgs')
 sclf = StackingCVClassifier(classifiers=[clf1, clf2], meta_classifier=lr, use_probas=True, cv=3)
 for clf, label in zip([clf1, clf2, clf3, sclf],
['KNN',
'Random Forest',
'Naive Bayes',
'StackingClassifier']):
scores = model_selection.cross_val_score(clf, train_numeric_X.values, train_numeric_Y[numeric_features_Y[0]].values, cv=3, scoring='neg_mean_squared_log_error')
 print("Avg_rmse: %0.2f(+/- %0.2f)[%s]" %(scores.mean() , scores.std() , label))
preds, target = learn.get_preds(DatasetType.Test, ordered=True)
 labels = preds.numpy()<save_to_csv>
train_y_pred = RF_model.predict(train_numeric_X )
test_id = test['id']
label_cols = ['toxic', 'severe_toxic' , 'obscene' , 'threat' , 'insult' , 'identity_hate']
submission = pd.DataFrame({'id': test_id})
 submission = pd.concat([submission, pd.DataFrame(preds.numpy() , columns = label_cols)], axis=1)
 submission.to_csv('submission.csv', index=False)
 submission.head()<load_from_csv>
error = np.sqrt(( train_y_pred - train_numeric_Y)**2)
 error = error.cumsum()
model_9872_baseline_submission = pd.read_csv(".. /input/model-9872/model_9872_baseline_submission.csv" )<load_from_csv>
rmse = mean_squared_error(train_numeric_Y, train_y_pred , squared=False)
 rmse
oof_model_9872_baseline_submission = pd.read_csv(".. /input/oof-model-9872/oof-model_9872_baseline_submission.csv" )<save_to_csv>
clf1 = RandomForestClassifier(n_estimators=1,n_jobs=4)
 clf3 = RandomForestClassifier(n_estimators=3,n_jobs=4)
 clf5 = RandomForestClassifier(n_estimators=5,n_jobs=4)
 clf10 = RandomForestClassifier(n_estimators=10,n_jobs=4)
 clf50 = RandomForestClassifier(n_estimators=50,n_jobs=4 )
model_9872_baseline_submission.to_csv('model_9872_baseline_submission.csv',index=False )<save_to_csv>
clf1.fit(train_numeric_X, train_numeric_Y)
 clf3.fit(train_numeric_X, train_numeric_Y)
 clf5.fit(train_numeric_X, train_numeric_Y)
 clf10.fit(train_numeric_X, train_numeric_Y)
 clf50.fit(train_numeric_X, train_numeric_Y )
oof_model_9872_baseline_submission.to_csv('oof-model_9872_baseline_submission.csv', index=False )<set_options>
predicted1 = clf1.predict(train_numeric_X)
 predicted3 = clf3.predict(train_numeric_X)
 predicted5 = clf5.predict(train_numeric_X)
 predicted10 = clf10.predict(train_numeric_X)
 predicted50 = clf50.predict(train_numeric_X )
print(tf.__version__)
 tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None
)<define_variables>
a = np.sum(( predicted1)-(train_numeric_Y)) **2 / len(predicted1)
 b = np.sum(( predicted3)-(train_numeric_Y)) **2 / len(predicted3)
 c = np.sum(( predicted5)-(train_numeric_Y)) **2 / len(predicted5)
 d = np.sum(( predicted10)-(train_numeric_Y)) **2 / len(predicted10)
 e = np.sum(( predicted50)-(train_numeric_Y)) **2 / len(predicted50 )
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
TEXT_COLUMN = 'comment_text'
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'<load_from_csv>
clf1 = RandomForestClassifier(n_estimators=10,n_jobs=4,max_depth=1)
 clf2 = RandomForestClassifier(n_estimators=10,n_jobs=4,max_depth=2)
 clf3 = RandomForestClassifier(n_estimators=10,n_jobs=4,max_depth=3)
 clf4 = RandomForestClassifier(n_estimators=10,n_jobs=4,max_depth=4)
 clf5 = RandomForestClassifier(n_estimators=10,n_jobs=4,max_depth=5)
 clf10 = RandomForestClassifier(n_estimators=10,n_jobs=4,max_depth=10 )
train_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv")
 y = train_df[list_classes].values
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = y
x_test = test_df[TEXT_COLUMN].astype(str )<categorify>
clf1.fit(train_numeric_X, train_numeric_Y)
 clf2.fit(train_numeric_X, train_numeric_Y)
 clf3.fit(train_numeric_X, train_numeric_Y)
 clf4.fit(train_numeric_X, train_numeric_Y)
 clf5.fit(train_numeric_X, train_numeric_Y)
 clf10.fit(train_numeric_X, train_numeric_Y )
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNGRU(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<string_transform>
predicted1 = clf1.predict(train_numeric_X)
 predicted2 = clf2.predict(train_numeric_X)
 predicted3 = clf3.predict(train_numeric_X)
 predicted4 = clf4.predict(train_numeric_X)
 predicted5 = clf5.predict(train_numeric_X)
 predicted10 = clf10.predict(train_numeric_X )
%%time
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<concatenate>
a = np.sum(( predicted1)-(train_numeric_Y)) **2 / len(predicted1)
 b = np.sum(( predicted2)-(train_numeric_Y)) **2 / len(predicted2)
 c = np.sum(( predicted3)-(train_numeric_Y)) **2 / len(predicted3)
 d = np.sum(( predicted4)-(train_numeric_Y)) **2 / len(predicted4)
 e = np.sum(( predicted5)-(train_numeric_Y)) **2 / len(predicted5)
 f = np.sum(( predicted10)-(train_numeric_Y)) **2 / len(predicted10 )
%%time
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )<split>
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from sklearn.naive_bayes import GaussianNB
X_train, X_valid, Y_train, Y_valid = train_test_split(x_train, y_train, test_size = 0.1)
<train_model>
pipeline = Pipeline([('scale', StandardScaler()),('pca', PCA(n_components=2)) ])
 pca = pipeline.fit(train_numeric_X)
 pca_X = pca.transform(train_numeric_X )
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 X_train,
Y_train,
validation_data =(X_valid, Y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 val_preds_3 = model.predict(X_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(Y_valid[:,i], val_preds_3[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5')
<save_to_csv>
from sklearn.manifold import TSNE
<import_modules><EOS> 
X_embedded = TSNE(n_components=3 ).fit_transform(train_numeric_X )