Kaggle competition: Natural Language Processing with Disaster Tweets
non_moa = train_features.iloc[labels_train.sum(axis=1)==0]
non_moa_stratifier = cp_type_train[labels_train.sum(axis=1)==0]<drop_column>
import math
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]<normalization>
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tokenizers import BertWordPieceTokenizer
from transformers import BertTokenizer, TFBertModel, BertConfig, TFBertForSequenceClassification
scaler = preprocessing.StandardScaler()
data_train = scaler.fit_transform(train_features)
 data_test = scaler.transform(test_features.drop('cp_type',axis=1))
non_moa = scaler.transform(non_moa )<define_variables>
def remove_url(raw_str):
clean_str = re.sub(r'http\S+', '', raw_str)
 return clean_str
n_features = data_train.shape[1]
n_labels = labels_train.shape[1]
n_train = data_train.shape[0]
n_test = data_test.shape[0]<categorify>
sample_submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 train_df = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test_df = pd.read_csv(".. /input/nlp-getting-started/test.csv" )
activation = 'swish'
encoding_dim = 1024
dropout = 0.01
non_moa_train, non_moa_test = train_test_split(non_moa,test_size=0.15)
 class Autoencoder(Model):
def __init__(self, n_features, encoding_dim):
super(Autoencoder, self ).__init__()
self.encoder = Sequential([
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(encoding_dim,activation=activation),
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(1.5*encoding_dim,activation=activation),
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(2*encoding_dim,activation=activation),
])
 self.decoder = Sequential([
layers.Dense(2*encoding_dim,activation=activation),
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(1.5*encoding_dim,activation=activation),
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(n_features)
 ])
 def call(self, x):
encoded = self.encoder(x)
 decoded = self.decoder(encoded)
 return decoded
autoencoder = Autoencoder(n_features,encoding_dim)
 autoencoder.compile(optimizer=optimizers.Adam(learning_rate=1E-4), loss='mse')
 ae_reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.3, patience=5, min_lr=1E-5)
 ae_early_stopping = callbacks.EarlyStopping(monitor='val_loss', min_delta=1E-5, patience=32, restore_best_weights=True)
 hist = autoencoder.fit(non_moa_train,non_moa_train,batch_size=32, verbose=0, validation_data =(non_moa_test,non_moa_test), epochs=1024,
shuffle=True, callbacks=[ae_reduce_lr, ae_early_stopping] )<categorify>
train_df.fillna('nan', inplace = True)
 test_df.fillna('nan', inplace = True )
autoencoded_train = autoencoder.encoder(data_train ).numpy()
autoencoded_test = autoencoder.encoder(data_test ).numpy()
autoencoded_non_moa_test = autoencoder.encoder(non_moa_test ).numpy()
autoencoder_error = mean_squared_error(non_moa_test,autoencoder.decoder(autoencoded_non_moa_test ).numpy())
 print("Autoencoder reconstruction error for Non MoA is " + str(autoencoder_error))
moa_train = data_train[labels_train.sum(axis=1)!=0]
autoencoder_error = mean_squared_error(moa_train,autoencoder.decoder(autoencoder.encoder(moa_train ).numpy() ).numpy())
 print("Autoencoder reconstruction error for train data with MoA is " + str(autoencoder_error))<compute_test_metric>
def clean(tweet):
tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let's", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"Ì©", "e", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"&gt;", ">", tweet)
 tweet = re.sub(r"&lt;", "<", tweet)
 tweet = re.sub(r"&amp;", "&", tweet)
 tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"8/5/2015", "2015-08-05", tweet)
 tweet = re.sub(r"chest/torso", "chest / torso", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"8/6/2015", "2015-08-06", tweet)
 tweet = re.sub(r"10:38PM", "10:38 PM", tweet)
 tweet = re.sub(r"10:30pm", "10:30 PM", tweet)
 tweet = re.sub(r"MH370:", "MH370 :", tweet)
 tweet = re.sub(r"PM:", "Prime Minister :", tweet)
 tweet = re.sub(r"Legionnaires:", "Legionnaires :", tweet)
 tweet = re.sub(r"Latest:", "Latest :", tweet)
 tweet = re.sub(r"Crash:", "Crash :", tweet)
 tweet = re.sub(r"News:", "News :", tweet)
 tweet = re.sub(r"derailment:", "derailment :", tweet)
 tweet = re.sub(r"attack:", "attack :", tweet)
 tweet = re.sub(r"Saipan:", "Saipan :", tweet)
 tweet = re.sub(r"Photo:", "Photo :", tweet)
 tweet = re.sub(r"Funtenna:", "Funtenna :", tweet)
 tweet = re.sub(r"quiz:", "quiz :", tweet)
 tweet = re.sub(r"VIDEO:", "VIDEO :", tweet)
 tweet = re.sub(r"MP:", "MP :", tweet)
 tweet = re.sub(r"UTC2015-08-05", "UTC 2015-08-05", tweet)
 tweet = re.sub(r"California:", "California :", tweet)
 tweet = re.sub(r"horror:", "horror :", tweet)
 tweet = re.sub(r"Past:", "Past :", tweet)
 tweet = re.sub(r"Time2015-08-06", "Time 2015-08-06", tweet)
 tweet = re.sub(r"here:", "here :", tweet)
 tweet = re.sub(r"fires.", "fires.", tweet)
 tweet = re.sub(r"Forest:", "Forest :", tweet)
 tweet = re.sub(r"Cramer:", "Cramer :", tweet)
 tweet = re.sub(r"Chile:", "Chile :", tweet)
 tweet = re.sub(r"link:", "link :", tweet)
 tweet = re.sub(r"crash:", "crash :", tweet)
 tweet = re.sub(r"Video:", "Video :", tweet)
 tweet = re.sub(r"Bestnaijamade:", "bestnaijamade :", tweet)
 tweet = re.sub(r"NWS:", "National Weather Service :", tweet)
 tweet = re.sub(r".caught", ".caught", tweet)
 tweet = re.sub(r"Hobbit:", "Hobbit :", tweet)
 tweet = re.sub(r"2015:", "2015 :", tweet)
 tweet = re.sub(r"post:", "post :", tweet)
 tweet = re.sub(r"BREAKING:", "BREAKING :", tweet)
 tweet = re.sub(r"Island:", "Island :", tweet)
 tweet = re.sub(r"Med:", "Med :", tweet)
 tweet = re.sub(r"97/Georgia", "97 / Georgia", tweet)
 tweet = re.sub(r"Here:", "Here :", tweet)
 tweet = re.sub(r"horror;", "horror ;", tweet)
 tweet = re.sub(r"people;", "people ;", tweet)
 tweet = re.sub(r"refugees;", "refugees ;", tweet)
 tweet = re.sub(r"Genocide;", "Genocide ;", tweet)
 tweet = re.sub(r".POTUS", ".POTUS", tweet)
 tweet = re.sub(r"Collision-No", "Collision - No", tweet)
 tweet = re.sub(r"Rear-", "Rear -", tweet)
 tweet = re.sub(r"Broadway:", "Broadway :", tweet)
 tweet = re.sub(r"Correction:", "Correction :", tweet)
 tweet = re.sub(r"UPDATE:", "UPDATE :", tweet)
 tweet = re.sub(r"Times:", "Times :", tweet)
 tweet = re.sub(r"RT:", "RT :", tweet)
 tweet = re.sub(r"Police:", "Police :", tweet)
 tweet = re.sub(r"Training:", "Training :", tweet)
 tweet = re.sub(r"Hawaii:", "Hawaii :", tweet)
 tweet = re.sub(r"Selfies:", "Selfies :", tweet)
 tweet = re.sub(r"Content:", "Content :", tweet)
 tweet = re.sub(r"101:", "101 :", tweet)
 tweet = re.sub(r"story:", "story :", tweet)
 tweet = re.sub(r"injured:", "injured :", tweet)
 tweet = re.sub(r"poll:", "poll :", tweet)
 tweet = re.sub(r"Guide:", "Guide :", tweet)
 tweet = re.sub(r"Update:", "Update :", tweet)
 tweet = re.sub(r"alarm:", "alarm :", tweet)
 tweet = re.sub(r"floods:", "floods :", tweet)
 tweet = re.sub(r"Flood:", "Flood :", tweet)
 tweet = re.sub(r"MH370;", "MH370 ;", tweet)
 tweet = re.sub(r"life:", "life :", tweet)
 tweet = re.sub(r"crush:", "crush :", tweet)
 tweet = re.sub(r"now:", "now :", tweet)
 tweet = re.sub(r"Vote:", "Vote :", tweet)
 tweet = re.sub(r"Catastrophe.", "Catastrophe.", tweet)
 tweet = re.sub(r"library:", "library :", tweet)
 tweet = re.sub(r"Bush:", "Bush :", tweet)
 tweet = re.sub(r";ACCIDENT", "; ACCIDENT", tweet)
 tweet = re.sub(r"accident:", "accident :", tweet)
 tweet = re.sub(r"Taiwan;", "Taiwan ;", tweet)
 tweet = re.sub(r"Map:", "Map :", tweet)
 tweet = re.sub(r"failure:", "failure :", tweet)
 tweet = re.sub(r"150-Foot", "150 - Foot", tweet)
 tweet = re.sub(r"failure:", "failure :", tweet)
 tweet = re.sub(r"prefer:", "prefer :", tweet)
 tweet = re.sub(r"CNN:", "CNN :", tweet)
 tweet = re.sub(r"Oops:", "Oops :", tweet)
 tweet = re.sub(r"Disco:", "Disco :", tweet)
 tweet = re.sub(r"Disease:", "Disease :", tweet)
 tweet = re.sub(r"Grows:", "Grows :", tweet)
 tweet = re.sub(r"projected:", "projected :", tweet)
 tweet = re.sub(r"Pakistan.", "Pakistan.", tweet)
 tweet = re.sub(r"ministers:", "ministers :", tweet)
 tweet = re.sub(r"Photos:", "Photos :", tweet)
 tweet = re.sub(r"Disease:", "Disease :", tweet)
 tweet = re.sub(r"pres:", "press :", tweet)
 tweet = re.sub(r"winds.", "winds.", tweet)
 tweet = re.sub(r"MPH.", "MPH.", tweet)
 tweet = re.sub(r"PHOTOS:", "PHOTOS :", tweet)
 tweet = re.sub(r"Time2015-08-05", "Time 2015-08-05", tweet)
 tweet = re.sub(r"Denmark:", "Denmark :", tweet)
 tweet = re.sub(r"Articles:", "Articles :", tweet)
 tweet = re.sub(r"Crash:", "Crash :", tweet)
 tweet = re.sub(r"casualties.:", "casualties.:", tweet)
 tweet = re.sub(r"Afghanistan:", "Afghanistan :", tweet)
 tweet = re.sub(r"Day:", "Day :", tweet)
 tweet = re.sub(r"AVERTED:", "AVERTED :", tweet)
 tweet = re.sub(r"sitting:", "sitting :", tweet)
 tweet = re.sub(r"Multiplayer:", "Multiplayer :", tweet)
 tweet = re.sub(r"Kaduna:", "Kaduna :", tweet)
 tweet = re.sub(r"favorite:", "favorite :", tweet)
 tweet = re.sub(r"home:", "home :", tweet)
 tweet = re.sub(r"just:", "just :", tweet)
 tweet = re.sub(r"Collision-1141", "Collision - 1141", tweet)
 tweet = re.sub(r"County:", "County :", tweet)
 tweet = re.sub(r"Duty:", "Duty :", tweet)
 tweet = re.sub(r"page:", "page :", tweet)
 tweet = re.sub(r"Attack:", "Attack :", tweet)
 tweet = re.sub(r"Minecraft:", "Minecraft :", tweet)
 tweet = re.sub(r"wounds;", "wounds ;", tweet)
 tweet = re.sub(r"Shots:", "Shots :", tweet)
 tweet = re.sub(r"shots:", "shots :", tweet)
 tweet = re.sub(r"Gunfire:", "Gunfire :", tweet)
 tweet = re.sub(r"hike:", "hike :", tweet)
 tweet = re.sub(r"Email:", "Email :", tweet)
 tweet = re.sub(r"System:", "System :", tweet)
 tweet = re.sub(r"Radio:", "Radio :", tweet)
 tweet = re.sub(r"King:", "King :", tweet)
 tweet = re.sub(r"upheaval:", "upheaval :", tweet)
 tweet = re.sub(r"tragedy;", "tragedy ;", tweet)
 tweet = re.sub(r"HERE:", "HERE :", tweet)
 tweet = re.sub(r"terrorism:", "terrorism :", tweet)
 tweet = re.sub(r"police:", "police :", tweet)
 tweet = re.sub(r"Mosque:", "Mosque :", tweet)
 tweet = re.sub(r"Rightways:", "Rightways :", tweet)
 tweet = re.sub(r"Brooklyn:", "Brooklyn :", tweet)
 tweet = re.sub(r"Arrived:", "Arrived :", tweet)
 tweet = re.sub(r"Home:", "Home :", tweet)
 tweet = re.sub(r"Earth:", "Earth :", tweet)
 tweet = re.sub(r"three:", "three :", tweet)
 tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"aRmageddon", "armageddon", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"withBioterrorism&use", "with Bioterrorism & use", tweet)
 tweet = re.sub(r"Hostage&2", "Hostage & 2", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"IDFire", "Idaho Fire", tweet)
 tweet = re.sub(r"DETECTADO", "Detected", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"NickCannon", "Nick Cannon", tweet)
 tweet = re.sub(r"FaroeIslands", "Faroe Islands", tweet)
 tweet = re.sub(r"yycstorm", "Calgary Storm", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 return tweet
pca = PCA(n_components=768)
 pca.fit(non_moa)
 pca_train = pca.transform(data_train)
 pca_test = pca.transform(data_test)
 pca_error_1 = mean_squared_error(non_moa,pca.inverse_transform(pca.transform(non_moa)))
 print('PCA Reconstruction Error for Non MoA is ' + str(pca_error_1))
pca_error = mean_squared_error(moa_train,pca.inverse_transform(pca.transform(moa_train)))
 print('PCA Reconstruction Error for test dataset is ' + str(pca_error))<categorify>
stopwords = ["they've", 'k', 'whom', "he'll", 'could', 'itself', 'hence', "when's", 'where', 'through',
'was', 'its', 'into', 'however', "she'd", "we'd", 'they', 'below', 'again', "she'll", "he's",
'did', 'my', 'are', 'our', "where's", 'above', 'ever', 'yourself', "i'd", 'just', 'we', "i'll",
'it', 'from', "we'll", 'that', 'he', 'cannot', "you're", 'her', 'this', "why's", 'once',
'am', 'ourselves', 'out', 'get', 'would', 'up', "it's", 'same', 'these', "you've", 'such', 'between',
'himself', 'also', "that's", 'r', 'www', 'all', "what's", 'if', 'http', 'herself', 'after', 'had',
'has', 'your', 'while', 'other', 'their', 'shall', 'more', 'off', 'as', 'hers', 'with', 'over',
'by', 'there', "we're", "they'll", 'any', 'to', 'no', 'about', 'both', "he'd", 'only', 'here',
'than', 'what', 'been', 'does', "we've", 'theirs', 'being', 'ought', "they'd", 'few', 'you', 'under',
'since', 'can', 'them', 'at', 'else', 'each', 'ours', 'therefore', 'most', 'before', 'then', 'his', 'me',
'a', 'further', "how's", 'during', 'of', 'like', 'on', 'themselves', 'why', 'those', 'in', 'too', 'she',
'because', "i've", "let's", 'how', 'very', "you'd", 'own', 'but', "she's", 'i', 'yourselves', 'down', 'should',
'and', 'do', 'or', 'were', 'some', 'an', "who's", 'otherwise', 'be', 'him', 'myself', 'have', 'which', "there's",
"i'm", 'when', 'doing', "you'll", 'com', 'for', 'who', 'yours', 'until', 'the', 'is', 'so', "they're", "here's",
'nor', 'having', 'will', 'may', 'one', 'now']
autoencoder_error_train = mean_squared_error(data_train.T,autoencoder.decoder(autoencoder.encoder(data_train ).numpy() ).numpy().T, multioutput='raw_values')
 autoencoder_error_test = mean_squared_error(data_test.T,autoencoder.decoder(autoencoder.encoder(data_test ).numpy() ).numpy().T, multioutput='raw_values')
 pca_error_train = mean_squared_error(data_train.T,pca.inverse_transform(pca.transform(data_train)).T, multioutput='raw_values')
 pca_error_test = mean_squared_error(data_test.T,pca.inverse_transform(pca.transform(data_test)).T, multioutput='raw_values')
 data_train = np.concatenate(( data_train,pca_train,autoencoded_train,autoencoder_error_train[:,np.newaxis],pca_error_train[:,np.newaxis]),axis=1)
 data_test = np.concatenate(( data_test,pca_test,autoencoded_test,autoencoder_error_test[:,np.newaxis],pca_error_test[:,np.newaxis]),axis=1 )<compute_train_metric>
def preprocessor2(text):
text = re.sub('[^a-zA-Z]', ' ', text)
 text = re.sub(r"\s+[a-zA-Z]\s+", ' ', text)
 text = re.sub(r'\s+', ' ', text)
 text = text.lower()
return text
pca_error_1 = mean_squared_error(non_moa_test.T,pca.inverse_transform(pca.transform(non_moa_test)).T, multioutput='raw_values')
 pca_error = mean_squared_error(moa_train.T,pca.inverse_transform(pca.transform(moa_train)).T, multioutput='raw_values')
 autoencoder_error_1 = mean_squared_error(non_moa_test.T,autoencoder.decoder(autoencoded_non_moa_test ).numpy().T, multioutput='raw_values')
 autoencoder_error = mean_squared_error(moa_train.T,autoencoder.decoder(autoencoder.encoder(moa_train ).numpy() ).numpy().T, multioutput='raw_values' )<compute_test_metric>
train_df['text'] = train_df['text'].apply(lambda x:remove_url(x))
n_labels = labels_train.shape[1]
n_features = data_train.shape[1]
n_train = data_train.shape[0]
n_test = data_test.shape[0]
p_min = 1*1E-3
p_max = 1-1*1E-3
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
n_seeds = 6
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_folds = 5
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 hists = []
for seed in seeds:
fold = 0
mskf = MultilabelStratifiedKFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in mskf.split(data_train,labels_train):
X_train = data_train[train]
X_test = data_train[test]
y_train = labels_train[train]
y_test = labels_train[test]
model = Sequential()
model.add(layers.Dropout(0.6))
model.add(layers.Dense(1024))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.6))
model.add(layers.Dense(256))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.1))
model.add(layers.Dense(n_labels))
model.add(layers.Activation('sigmoid'))
model.compile(optimizer=optimizers.Adam(learning_rate=1*1E-2), loss=losses.BinaryCrossentropy(label_smoothing=0.001), metrics=logloss)
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.2, patience=5, mode='min', min_lr=1E-5)
 early_stopping = callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=15, mode='min',restore_best_weights=True)
 hist = model.fit(X_train,y_train, batch_size=128, epochs=192,verbose=0,validation_data =(X_test,y_test),callbacks=[reduce_lr, early_stopping])
 hists.append(hist)
 y_val = model.predict(X_test)
 oof += logloss(tf.constant(y_test,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 y_pred += model.predict(data_test)/(n_folds*n_seeds)
 fold += 1 
<save_to_csv>
train_df['merged'] = train_df['text'].astype(str)+' '+train_df['keyword']+' '+train_df['location']
test_df['merged'] = test_df['text'].astype(str)+' '+test_df['keyword']+' '+test_df['location']
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission.csv', index=False )<set_options>
train_df['merged_cleaned'] = train_df['merged'].copy().apply(lambda x : clean(str(x)) ).apply(lambda x : preprocessor2(x)).apply(lambda x: ' '.join([word for word in x.split() if word not in(stopwords)]))
test_df['merged_cleaned'] = test_df['merged'].copy().apply(lambda x : clean(str(x)) ).apply(lambda x : preprocessor2(x)).apply(lambda x: ' '.join([word for word in x.split() if word not in(stopwords)]))
warnings.filterwarnings('ignore' )<load_from_csv>
train_df['totalwords'] = train_df['merged_cleaned'].str.count(' ')+ 1
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
max_len = 50
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<set_options>
train_df = train_df[['merged_cleaned', 'target']]
test_df = test_df[['id','merged_cleaned']]
def seed_everything(seed=1903):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<concatenate>
label_L = list(train_df.target )
n_comp = 28
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 5
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
configuration = BertConfig.from_pretrained(f'/kaggle/input/bert-tensorflow/bert-base-uncased-config.json' )
var_thresh = VarianceThreshold(threshold=0.4)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features<merge>
slow_tokenizer = BertTokenizer.from_pretrained(f'/kaggle/input/bertbaseuncased/vocab.txt')
 save_path = "bert-base-uncased/"
if not os.path.exists(save_path):
os.makedirs(save_path)
 slow_tokenizer.save_pretrained(save_path)
 tokenizer = BertWordPieceTokenizer(f'/kaggle/input/bertbaseuncased/vocab.txt', lowercase=True)
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
class SquadExample:
def __init__(self, text):
self.text = text
self.skip = False
def preprocess(self):
text = self.text
text = str(text)
 tokenized_context = tokenizer.encode(text)
 input_ids = tokenized_context.ids
token_type_ids = [0] * len(tokenized_context.ids)
 attention_mask = [1] * len(input_ids)
 padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([0] * padding_length)
 attention_mask = attention_mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 elif padding_length < 0:
self.skip = True
self.input_ids = input_ids
self.token_type_ids = token_type_ids
self.attention_mask = attention_mask
def create_examples(raw_data):
squad_examples = []
for i in range(raw_data.shape[0]):
text = raw_data["merged_cleaned"][i]
squad_eg = SquadExample(text)
 squad_eg.preprocess()
squad_examples.append(squad_eg)
 return squad_examples
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<split>
def create_inputs_targets(squad_examples,label):
dataset_dict = {
"input_ids": [],
"token_type_ids": [],
"attention_mask": [],
}
for item in squad_examples:
if item.skip == False:
for key in dataset_dict:
dataset_dict[key].append(getattr(item, key))
for key in dataset_dict:
dataset_dict[key] = np.array(dataset_dict[key])
 x = [
dataset_dict["input_ids"],
dataset_dict["token_type_ids"],
dataset_dict["attention_mask"],
]
y = [np.array(label)]
return x, y
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int )<categorify>
train_examples = create_examples(train_df )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
test_examples = create_examples(test_df )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<normalization>
x_train, y_train = create_inputs_targets(train_examples,label_L )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.leaky_relu(self.dense1(x), 1e-3)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<feature_engineering>
x_test, y_test = create_inputs_targets(test_examples,None )
def process_data(df):
df['time_dose'] = df['cp_time'].astype(str)+df['cp_dose']
df = pd.get_dummies(df, columns=['cp_time','cp_dose','time_dose'])
 features_g = list(df.columns[4:776])
 features_c = list(df.columns[776:876])
 df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 features_c = list(df.columns[776:876])
 for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
return df<define_variables>
print(f"{len(train_examples)} training points created." )
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048<prepare_x_and_y>
base_path = '/kaggle/input/bert-tensorflow/bert-base-uncased-tf_model.h5'
def create_model(path = base_path):
encoder = TFBertForSequenceClassification.from_pretrained(path, config=configuration)
 input_ids = layers.Input(shape=(max_len,), dtype=tf.int32)
 token_type_ids = layers.Input(shape=(max_len,), dtype=tf.int32)
 attention_mask = layers.Input(shape=(max_len,), dtype=tf.int32)
 embedding = encoder(
 input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask 
)[0]
reshape_L = layers.Reshape(( 2,1))(embedding)
 lstm = layers.LSTM(64 )(reshape_L)
 label_class_0 = layers.Dense(128, name="process-1", use_bias=False )(lstm)
 label_class_1 = layers.Dense(128, name="process-2", use_bias=False )(label_class_0)
 label_class = layers.Dense(2, name="label_class", use_bias=False )(label_class_1)
 label_class = layers.Flatten()(label_class)
 label_probs = layers.Activation(keras.activations.softmax )(label_class)
 model = keras.Model(
 inputs=[input_ids, token_type_ids, attention_mask],
outputs=[label_probs], 
)
 loss = keras.losses.SparseCategoricalCrossentropy(from_logits=False)
 optimizer = keras.optimizers.Adam(lr=2e-5)
 model.compile(optimizer=optimizer, loss=[loss, loss])
 return model
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
print(f"FOLD: {fold}")
 for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"EPOCH: {epoch}, train_loss: {train_loss}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
model.fit(
 x_train,
y_train,
epochs=3,
verbose=2,
batch_size=16
)
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<feature_engineering>
Y_pred = model.predict(x_test )
SEED = [1903, 1881, 2020]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
print(f"SEED: {seed}")
 oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions<merge>
Y_pred = np.argmax(Y_pred,axis=1 )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<save_to_csv>
pred_df = pd.DataFrame(Y_pred, columns=['target'])
 result = pd.concat([test_df,pred_df], axis=1, join='outer', ignore_index=False, keys=None, sort = False)
 result = result[['id','target']]
print(Y_pred )
<install_modules><EOS> 
result.to_csv('sample_submission2.csv',index=False )