Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
%matplotlib inline
warnings.filterwarnings('ignore' )
EMBEDDING_FILE = '.. /input/glove-twitter/glove.twitter.27B.100d.txt'<feature_engineering>
df_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df_train.head()
all_embs = np.stack(list(embeddings_index.values()))
emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<define_variables>
df_train.drop('Id',axis=1,inplace=True )
batch_size = 1024
epochs = 15
embed_size = 100<choose_model_class>
def visualNA(df,perc=0):
NAN = [(c, df[c].isna().mean() *100)for c in df]
NAN = pd.DataFrame(NAN, columns=["column_name", "percentage"])
 NAN = NAN[NAN.percentage > perc]
print(NAN.sort_values("percentage", ascending=False))
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', patience = 2, verbose=1,factor=0.5, min_lr=0.00001 )<choose_model_class>
def handleNA(df):
df['Alley'].fillna(value='No alley access',inplace=True)
 df['BsmtQual'].fillna(value='No Basement',inplace=True)
 df['BsmtCond'].fillna(value='No Basement',inplace=True)
 df['BsmtExposure'].fillna(value='No Basement',inplace=True)
 df['BsmtFinType1'].fillna(value='No Basement',inplace=True)
 df['BsmtFinType2'].fillna(value='No Basement',inplace=True)
 df['FireplaceQu'].fillna(value='No Fireplace',inplace=True)
 df['GarageType'].fillna(value='No Garage',inplace=True)
 df['GarageYrBlt'].fillna(value=0,inplace=True)
 df['GarageFinish'].fillna(value='No Garage',inplace=True)
 df['GarageQual'].fillna(value='No Garage',inplace=True)
 df['GarageCond'].fillna(value='No Garage',inplace=True)
 df['MasVnrType'].fillna(value='None',inplace=True)
 df['MasVnrArea'].fillna(value=0.0,inplace=True)
 df['PoolQC'].fillna(value='No Pool',inplace=True)
 df['Fence'].fillna(value='No Fence',inplace=True)
 df['MiscFeature'].fillna(value='None',inplace=True)
model = Sequential()
model.add(Embedding(max_features, output_dim=embed_size, weights=[embedding_matrix], input_length=max_len, trainable=False))
model.add(LSTM(units=128 , return_sequences = False , recurrent_dropout = 0.3 , dropout = 0.3))
model.add(Dense(units=64 , activation = 'relu', kernel_regularizer='l2'))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer=keras.optimizers.Adam(lr = 0.01), loss='binary_crossentropy', metrics=['accuracy'] )<train_model>
df_train[df_train['Electrical'].isnull() ]
history = model.fit(X_train, y_train, batch_size = batch_size ,
validation_data =(X_dev,y_dev),
epochs = epochs , callbacks = [learning_rate_reduction] )<compute_test_metric>
df_train[(df_train['Electrical'].notnull())&(df_train['MSSubClass']==80)
 &(df_train['LotFrontage']==73)]['Electrical']
print("Accuracy of the model on Training Data is - " , model.evaluate(X_train,y_train)[1]*100)
 print("Accuracy of the model on Dev Data is - " , model.evaluate(X_dev,y_dev)[1]*100 )<prepare_x_and_y>
df_train.loc[df_train['Electrical'].isnull() ,'Electrical'] = 'SBrkr'
X_test = test_df.text.values<string_transform>
dict_neighbor = {
'NAmes' :{'lat': 42.045830,'lon': -93.620767},
'CollgCr':{'lat': 42.018773,'lon': -93.685543},
'OldTown':{'lat': 42.030152,'lon': -93.614628},
'Edwards':{'lat': 42.021756,'lon': -93.670324},
'Somerst':{'lat': 42.050913,'lon': -93.644629},
'Gilbert':{'lat': 42.060214,'lon': -93.643179},
'NridgHt':{'lat': 42.060357,'lon': -93.655263},
'Sawyer' :{'lat': 42.034446,'lon': -93.666330},
'NWAmes' :{'lat': 42.049381,'lon': -93.634993},
'SawyerW':{'lat': 42.033494,'lon': -93.684085},
'BrkSide':{'lat': 42.032422,'lon': -93.626037},
'Crawfor':{'lat': 42.015189,'lon': -93.644250},
'Mitchel':{'lat': 41.990123,'lon': -93.600964},
'NoRidge':{'lat': 42.051748,'lon': -93.653524},
'Timber' :{'lat': 41.998656,'lon': -93.652534},
'IDOTRR' :{'lat': 42.022012,'lon': -93.622183},
'ClearCr':{'lat': 42.060021,'lon': -93.629193},
'StoneBr':{'lat': 42.060227,'lon': -93.633546},
'SWISU' :{'lat': 42.022646,'lon': -93.644853},
'MeadowV':{'lat': 41.991846,'lon': -93.603460},
'Blmngtn':{'lat': 42.059811,'lon': -93.638990},
'BrDale' :{'lat': 42.052792,'lon': -93.628820},
'Veenker':{'lat': 42.040898,'lon': -93.651502},
'NPkVill':{'lat': 42.049912,'lon': -93.626546},
'Blueste':{'lat': 42.010098,'lon': -93.647269}
}
tokenized_dev = tokenizer.texts_to_sequences(X_test)
 X_test = sequence.pad_sequences(tokenized_dev, maxlen=max_len )<predict_on_test>
df_train['Lat'] = df_train['Neighborhood'].map(lambda neighbor: dict_neighbor[neighbor]['lat'])
 df_train['Lon'] = df_train['Neighborhood'].map(lambda neighbor: dict_neighbor[neighbor]['lon'] )
classes = model.predict_classes(X_test)[:, 0]<create_dataframe>
Categorical_features = df_train.select_dtypes(include=['object'])
 Numerical_features = df_train.select_dtypes(exclude=['object'] )
submission = pd.DataFrame(
 {'id': list(test_df.index.values),
'target': list(classes),
} ).set_index('id' )<save_to_csv>
X = df_train.drop('SalePrice',axis=1)
 y = df_train['SalePrice']
submission.to_csv('submission.csv' )<set_options>
LotFrontageX = X['LotFrontage']
GarageYrBltX = X['GarageYrBlt']
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000 )<load_from_csv>
ohe = OneHotEncoder(sparse=False,drop="if_binary")
 Categorical_Encoded = ohe.fit_transform(Categorical_features.astype(str))
train = pd.read_csv('.. /input/nlp-getting-started/train.csv',)
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )<feature_engineering>
Categorical_Encoded_Frame = pd.DataFrame(Categorical_Encoded, columns= ohe.get_feature_names(Categorical_features.columns))
Categorical_Encoded_Frame.head()
train['word_count'] = train['text'].apply(lambda x: len(str(x ).split()))
test['word_count'] = test['text'].apply(lambda x: len(str(x ).split()))
train['unique_word_count'] = train['text'].apply(lambda x: len(set(str(x ).split())))
 test['unique_word_count'] = test['text'].apply(lambda x: len(set(str(x ).split())))
 train['stop_word_count'] = train['text'].apply(lambda x: len([w for w in str(x ).lower().split() if w in STOPWORDS]))
test['stop_word_count'] = test['text'].apply(lambda x: len([w for w in str(x ).lower().split() if w in STOPWORDS]))
train['url_count'] = train['text'].apply(lambda x: len([w for w in str(x ).lower().split() if 'http' in w or 'https' in w]))
test['url_count'] = test['text'].apply(lambda x: len([w for w in str(x ).lower().split() if 'http' in w or 'https' in w]))
train['mean_word_len'] = train['text'].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
test['mean_word_len'] = test['text'].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
train['char_count'] = train['text'].apply(lambda x: len(str(x)))
 test['char_count'] = test['text'].apply(lambda x: len(str(x)))
 train['punct_count'] = train['text'].apply(lambda x: len([w for w in str(x)if w in string.punctuation]))
test['punct_count'] = test['text'].apply(lambda x: len([w for w in str(x)if w in string.punctuation]))
train['hashtag_count'] = train['text'].apply(lambda x: len([w for w in str(x)if w == '
test['hashtag_count'] = test['text'].apply(lambda x: len([w for w in str(x)if w == '
train['mention_count'] = train['text'].apply(lambda x: len([w for w in str(x)if w == '@']))
test['mention_count'] = test['text'].apply(lambda x: len([w for w in str(x)if w == '@'])) 
<feature_engineering>
Numerical_features_X = Numerical_features.drop(['SalePrice','LotFrontage','GarageYrBlt'],axis=1 )
def generate_ngrams(text,ngram=1, n=None):
vec = CountVectorizer(ngram_range=(ngram, ngram)).fit(text)
 bag_of_words = vec.transform(text)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq[:n] 
<feature_engineering>
X = Categorical_Encoded_Frame.join(Numerical_features_X ).join(LotFrontageX ).join(GarageYrBltX)
 Xcolumns = X.columns
X.head()
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 train['text'] = train['text'].apply(lambda x: remove_URL(x))
test['text'] = test['text'].apply(lambda x: remove_URL(x))<feature_engineering>
X = KNNImputer(n_neighbors=5 ).fit_transform(X )
PUNCT_TO_REMOVE = string.punctuation
def remove_punctuation(text):
return text.translate(str.maketrans(' ',' ',PUNCT_TO_REMOVE))
train['text'] = train['text'].apply(lambda text: remove_punctuation(text))
test['text'] = test['text'].apply(lambda text: remove_punctuation(text))<feature_engineering>
X = pd.DataFrame(X,columns=Xcolumns)
 X.head()
def remove_special_char(text):
text = text.replace('\r',' ')
 text = text.replace('
',' ')
 text = text.replace('\t',' ')
 return text
train['text'] = train['text'].apply(lambda x: remove_special_char(x))
test['text'] = test['text'].apply(lambda x: remove_special_char(x))<categorify>
y_log = np.log(y )
def clean(tweet):
tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let's", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"Ì©", "e", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"åÇ", "", tweet)
 tweet = re.sub(r"å£3million", "3 million", tweet)
 tweet = re.sub(r"åÀ", "", tweet)
 tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 tweet = re.sub(r"&gt;", ">", tweet)
 tweet = re.sub(r"&lt;", "<", tweet)
 tweet = re.sub(r"&amp;", "&", tweet)
 tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"amirite", "am I right", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"8/5/2015", "2015-08-05", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"8/6/2015", "2015-08-06", tweet)
 tweet = re.sub(r"10:38PM", "10:38 PM", tweet)
 tweet = re.sub(r"10:30pm", "10:30 PM", tweet)
 tweet = re.sub(r"16yr", "16 year", tweet)
 tweet = re.sub(r"lmao", "laughing my ass off", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"aRmageddon", "armageddon", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"withBioterrorism&use", "with Bioterrorism & use", tweet)
 tweet = re.sub(r"Hostage&2", "Hostage & 2", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"IDFire", "Idaho Fire", tweet)
 tweet = re.sub(r"DETECTADO", "Detected", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"NickCannon", "Nick Cannon", tweet)
 tweet = re.sub(r"FaroeIslands", "Faroe Islands", tweet)
 tweet = re.sub(r"yycstorm", "Calgary Storm", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 tweet = re.sub(r"justinbieber", "justin bieber", tweet)
 tweet = re.sub(r"UTC2015", "UTC 2015", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"djicemoon", "dj icemoon", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"thisiswhywecanthavenicethings", "this is why we cannot have nice things", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"c4news", "c4 news", tweet)
 tweet = re.sub(r"OBLITERATION", "obliteration", tweet)
 tweet = re.sub(r"MUDSLIDE", "mudslide", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"NotExplained", "Not Explained", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"LiveOnK2", "Live On K2", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"nikeplus", "nike plus", tweet)
 tweet = re.sub(r"david_cameron", "David Cameron", tweet)
 tweet = re.sub(r"peterjukes", "Peter Jukes", tweet)
 tweet = re.sub(r"JamesMelville", "James Melville", tweet)
 tweet = re.sub(r"megynkelly", "Megyn Kelly", tweet)
 tweet = re.sub(r"cnewslive", "C News Live", tweet)
 tweet = re.sub(r"JamaicaObserver", "Jamaica Observer", tweet)
 tweet = re.sub(r"TweetLikeItsSeptember11th2001", "Tweet like it is september 11th 2001", tweet)
 tweet = re.sub(r"cbplawyers", "cbp lawyers", tweet)
 tweet = re.sub(r"fewmoretweets", "few more tweets", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"cjoyner", "Chris Joyner", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"ScottWalker", "Scott Walker", tweet)
 tweet = re.sub(r"MikeParrActor", "Michael Parr", tweet)
 tweet = re.sub(r"4PlayThursdays", "Foreplay Thursdays", tweet)
 tweet = re.sub(r"TGF2015", "Tontitown Grape Festival", tweet)
 tweet = re.sub(r"realmandyrain", "Mandy Rain", tweet)
 tweet = re.sub(r"GraysonDolan", "Grayson Dolan", tweet)
 tweet = re.sub(r"ApolloBrown", "Apollo Brown", tweet)
 tweet = re.sub(r"saddlebrooke", "Saddlebrooke", tweet)
 tweet = re.sub(r"TontitownGrape", "Tontitown Grape", tweet)
 tweet = re.sub(r"AbbsWinston", "Abbs Winston", tweet)
 tweet = re.sub(r"ShaunKing", "Shaun King", tweet)
 tweet = re.sub(r"MeekMill", "Meek Mill", tweet)
 tweet = re.sub(r"TornadoGiveaway", "Tornado Giveaway", tweet)
 tweet = re.sub(r"GRupdates", "GR updates", tweet)
 tweet = re.sub(r"SouthDowns", "South Downs", tweet)
 tweet = re.sub(r"braininjury", "brain injury", tweet)
 tweet = re.sub(r"auspol", "Australian politics", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"weallheartonedirection", "we all heart one direction", tweet)
 tweet = re.sub(r"edsheeran", "Ed Sheeran", tweet)
 tweet = re.sub(r"TrueHeroes", "True Heroes", tweet)
 tweet = re.sub(r"S3XLEAK", "sex leak", tweet)
 tweet = re.sub(r"ComplexMag", "Complex Magazine", tweet)
 tweet = re.sub(r"TheAdvocateMag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"CityofCalgary", "City of Calgary", tweet)
 tweet = re.sub(r"EbolaOutbreak", "Ebola Outbreak", tweet)
 tweet = re.sub(r"SummerFate", "Summer Fate", tweet)
 tweet = re.sub(r"RAmag", "Royal Academy Magazine", tweet)
 tweet = re.sub(r"offers2go", "offers to go", tweet)
 tweet = re.sub(r"foodscare", "food scare", tweet)
 tweet = re.sub(r"MNPDNashville", "Metropolitan Nashville Police Department", tweet)
 tweet = re.sub(r"TfLBusAlerts", "TfL Bus Alerts", tweet)
 tweet = re.sub(r"GamerGate", "Gamer Gate", tweet)
 tweet = re.sub(r"IHHen", "Humanitarian Relief", tweet)
 tweet = re.sub(r"spinningbot", "spinning bot", tweet)
 tweet = re.sub(r"ModiMinistry", "Modi Ministry", tweet)
 tweet = re.sub(r"TAXIWAYS", "taxi ways", tweet)
 tweet = re.sub(r"Calum5SOS", "Calum Hood", tweet)
 tweet = re.sub(r"po_st", "po.st", tweet)
 tweet = re.sub(r"scoopit", "scoop.it", tweet)
 tweet = re.sub(r"UltimaLucha", "Ultima Lucha", tweet)
 tweet = re.sub(r"JonathanFerrell", "Jonathan Ferrell", tweet)
 tweet = re.sub(r"aria_ahrary", "Aria Ahrary", tweet)
 tweet = re.sub(r"rapidcity", "Rapid City", tweet)
 tweet = re.sub(r"OutBid", "outbid", tweet)
 tweet = re.sub(r"lavenderpoetrycafe", "lavender poetry cafe", tweet)
 tweet = re.sub(r"EudryLantiqua", "Eudry Lantiqua", tweet)
 tweet = re.sub(r"15PM", "15 PM", tweet)
 tweet = re.sub(r"OriginalFunko", "Funko", tweet)
 tweet = re.sub(r"rightwaystan", "Richard Tan", tweet)
 tweet = re.sub(r"CindyNoonan", "Cindy Noonan", tweet)
 tweet = re.sub(r"RT_America", "RT America", tweet)
 tweet = re.sub(r"narendramodi", "Narendra Modi", tweet)
 tweet = re.sub(r"BakeOffFriends", "Bake Off Friends", tweet)
 tweet = re.sub(r"TeamHendrick", "Hendrick Motorsports", tweet)
 tweet = re.sub(r"alexbelloli", "Alex Belloli", tweet)
 tweet = re.sub(r"itsjustinstuart", "Justin Stuart", tweet)
 tweet = re.sub(r"gunsense", "gun sense", tweet)
 tweet = re.sub(r"DebateQuestionsWeWantToHear", "debate questions we want to hear", tweet)
 tweet = re.sub(r"RoyalCarribean", "Royal Carribean", tweet)
 tweet = re.sub(r"samanthaturne19", "Samantha Turner", tweet)
 tweet = re.sub(r"JonVoyage", "Jon Stewart", tweet)
 tweet = re.sub(r"renew911health", "renew 911 health", tweet)
 tweet = re.sub(r"SuryaRay", "Surya Ray", tweet)
 tweet = re.sub(r"pattonoswalt", "Patton Oswalt", tweet)
 tweet = re.sub(r"minhazmerchant", "Minhaz Merchant", tweet)
 tweet = re.sub(r"TLVFaces", "Israel Diaspora Coalition", tweet)
 tweet = re.sub(r"pmarca", "Marc Andreessen", tweet)
 tweet = re.sub(r"pdx911", "Portland Police", tweet)
 tweet = re.sub(r"jamaicaplain", "Jamaica Plain", tweet)
 tweet = re.sub(r"Japton", "Arkansas", tweet)
 tweet = re.sub(r"RouteComplex", "Route Complex", tweet)
 tweet = re.sub(r"INSubcontinent", "Indian Subcontinent", tweet)
 tweet = re.sub(r"NJTurnpike", "New Jersey Turnpike", tweet)
 tweet = re.sub(r"Politifiact", "PolitiFact", tweet)
 tweet = re.sub(r"Hiroshima70", "Hiroshima", tweet)
 tweet = re.sub(r"GMMBC", "Greater Mt Moriah Baptist Church", tweet)
 tweet = re.sub(r"versethe", "verse the", tweet)
 tweet = re.sub(r"TubeStrike", "Tube Strike", tweet)
 tweet = re.sub(r"MissionHills", "Mission Hills", tweet)
 tweet = re.sub(r"ProtectDenaliWolves", "Protect Denali Wolves", tweet)
 tweet = re.sub(r"NANKANA", "Nankana", tweet)
 tweet = re.sub(r"SAHIB", "Sahib", tweet)
 tweet = re.sub(r"PAKPATTAN", "Pakpattan", tweet)
 tweet = re.sub(r"Newz_Sacramento", "News Sacramento", tweet)
 tweet = re.sub(r"gofundme", "go fund me", tweet)
 tweet = re.sub(r"pmharper", "Stephen Harper", tweet)
 tweet = re.sub(r"IvanBerroa", "Ivan Berroa", tweet)
 tweet = re.sub(r"LosDelSonido", "Los Del Sonido", tweet)
 tweet = re.sub(r"bancodeseries", "banco de series", tweet)
 tweet = re.sub(r"timkaine", "Tim Kaine", tweet)
 tweet = re.sub(r"IdentityTheft", "Identity Theft", tweet)
 tweet = re.sub(r"AllLivesMatter", "All Lives Matter", tweet)
 tweet = re.sub(r"mishacollins", "Misha Collins", tweet)
 tweet = re.sub(r"BillNeelyNBC", "Bill Neely", tweet)
 tweet = re.sub(r"BeClearOnCancer", "be clear on cancer", tweet)
 tweet = re.sub(r"Kowing", "Knowing", tweet)
 tweet = re.sub(r"ScreamQueens", "Scream Queens", tweet)
 tweet = re.sub(r"AskCharley", "Ask Charley", tweet)
 tweet = re.sub(r"BlizzHeroes", "Heroes of the Storm", tweet)
 tweet = re.sub(r"BradleyBrad47", "Bradley Brad", tweet)
 tweet = re.sub(r"HannaPH", "Typhoon Hanna", tweet)
 tweet = re.sub(r"meinlcymbals", "MEINL Cymbals", tweet)
 tweet = re.sub(r"Ptbo", "Peterborough", tweet)
 tweet = re.sub(r"cnnbrk", "CNN Breaking News", tweet)
 tweet = re.sub(r"IndianNews", "Indian News", tweet)
 tweet = re.sub(r"savebees", "save bees", tweet)
 tweet = re.sub(r"GreenHarvard", "Green Harvard", tweet)
 tweet = re.sub(r"StandwithPP", "Stand with planned parenthood", tweet)
 tweet = re.sub(r"hermancranston", "Herman Cranston", tweet)
 tweet = re.sub(r"WMUR9", "WMUR-TV", tweet)
 tweet = re.sub(r"RockBottomRadFM", "Rock Bottom Radio", tweet)
 tweet = re.sub(r"ameenshaikh3", "Ameen Shaikh", tweet)
 tweet = re.sub(r"ProSyn", "Project Syndicate", tweet)
 tweet = re.sub(r"Daesh", "ISIS", tweet)
 tweet = re.sub(r"s2g", "swear to god", tweet)
 tweet = re.sub(r"listenlive", "listen live", tweet)
 tweet = re.sub(r"CDCgov", "Centers for Disease Control and Prevention", tweet)
 tweet = re.sub(r"FoxNew", "Fox News", tweet)
 tweet = re.sub(r"CBSBigBrother", "Big Brother", tweet)
 tweet = re.sub(r"JulieDiCaro", "Julie DiCaro", tweet)
 tweet = re.sub(r"theadvocatemag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"RohnertParkDPS", "Rohnert Park Police Department", tweet)
 tweet = re.sub(r"THISIZBWRIGHT", "Bonnie Wright", tweet)
 tweet = re.sub(r"Popularmmos", "Popular MMOs", tweet)
 tweet = re.sub(r"WildHorses", "Wild Horses", tweet)
 tweet = re.sub(r"FantasticFour", "Fantastic Four", tweet)
 tweet = re.sub(r"HORNDALE", "Horndale", tweet)
 tweet = re.sub(r"PINER", "Piner", tweet)
 tweet = re.sub(r"BathAndNorthEastSomerset", "Bath and North East Somerset", tweet)
 tweet = re.sub(r"thatswhatfriendsarefor", "that is what friends are for", tweet)
 tweet = re.sub(r"residualincome", "residual income", tweet)
 tweet = re.sub(r"YahooNewsDigest", "Yahoo News Digest", tweet)
 tweet = re.sub(r"MalaysiaAirlines", "Malaysia Airlines", tweet)
 tweet = re.sub(r"AmazonDeals", "Amazon Deals", tweet)
 tweet = re.sub(r"MissCharleyWebb", "Charley Webb", tweet)
 tweet = re.sub(r"shoalstraffic", "shoals traffic", tweet)
 tweet = re.sub(r"GeorgeFoster72", "George Foster", tweet)
 tweet = re.sub(r"pop2015", "pop 2015", tweet)
 tweet = re.sub(r"_PokemonCards_", "Pokemon Cards", tweet)
 tweet = re.sub(r"DianneG", "Dianne Gallagher", tweet)
 tweet = re.sub(r"KashmirConflict", "Kashmir Conflict", tweet)
 tweet = re.sub(r"BritishBakeOff", "British Bake Off", tweet)
 tweet = re.sub(r"FreeKashmir", "Free Kashmir", tweet)
 tweet = re.sub(r"mattmosley", "Matt Mosley", tweet)
 tweet = re.sub(r"BishopFred", "Bishop Fred", tweet)
 tweet = re.sub(r"EndConflict", "End Conflict", tweet)
 tweet = re.sub(r"EndOccupation", "End Occupation", tweet)
 tweet = re.sub(r"UNHEALED", "unhealed", tweet)
 tweet = re.sub(r"CharlesDagnall", "Charles Dagnall", tweet)
 tweet = re.sub(r"Latestnews", "Latest news", tweet)
 tweet = re.sub(r"KindleCountdown", "Kindle Countdown", tweet)
 tweet = re.sub(r"NoMoreHandouts", "No More Handouts", tweet)
 tweet = re.sub(r"datingtips", "dating tips", tweet)
 tweet = re.sub(r"charlesadler", "Charles Adler", tweet)
 tweet = re.sub(r"twia", "Texas Windstorm Insurance Association", tweet)
 tweet = re.sub(r"txlege", "Texas Legislature", tweet)
 tweet = re.sub(r"WindstormInsurer", "Windstorm Insurer", tweet)
 tweet = re.sub(r"Newss", "News", tweet)
 tweet = re.sub(r"hempoil", "hemp oil", tweet)
 tweet = re.sub(r"CommoditiesAre", "Commodities are", tweet)
 tweet = re.sub(r"tubestrike", "tube strike", tweet)
 tweet = re.sub(r"JoeNBC", "Joe Scarborough", tweet)
 tweet = re.sub(r"LiteraryCakes", "Literary Cakes", tweet)
 tweet = re.sub(r"TI5", "The International 5", tweet)
 tweet = re.sub(r"thehill", "the hill", tweet)
 tweet = re.sub(r"3others", "3 others", tweet)
 tweet = re.sub(r"stighefootball", "Sam Tighe", tweet)
 tweet = re.sub(r"whatstheimportantvideo", "what is the important video", tweet)
 tweet = re.sub(r"ClaudioMeloni", "Claudio Meloni", tweet)
 tweet = re.sub(r"DukeSkywalker", "Duke Skywalker", tweet)
 tweet = re.sub(r"carsonmwr", "Fort Carson", tweet)
 tweet = re.sub(r"offdishduty", "off dish duty", tweet)
 tweet = re.sub(r"andword", "and word", tweet)
 tweet = re.sub(r"rhodeisland", "Rhode Island", tweet)
 tweet = re.sub(r"easternoregon", "Eastern Oregon", tweet)
 tweet = re.sub(r"WAwildfire", "Washington Wildfire", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"57am", "57 am", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"JacobHoggard", "Jacob Hoggard", tweet)
 tweet = re.sub(r"newnewnew", "new new new", tweet)
 tweet = re.sub(r"under50", "under 50", tweet)
 tweet = re.sub(r"getitbeforeitsgone", "get it before it is gone", tweet)
 tweet = re.sub(r"freshoutofthebox", "fresh out of the box", tweet)
 tweet = re.sub(r"amwriting", "am writing", tweet)
 tweet = re.sub(r"Bokoharm", "Boko Haram", tweet)
 tweet = re.sub(r"Nowlike", "Now like", tweet)
 tweet = re.sub(r"seasonfrom", "season from", tweet)
 tweet = re.sub(r"epicente", "epicenter", tweet)
 tweet = re.sub(r"epicenterr", "epicenter", tweet)
 tweet = re.sub(r"sicklife", "sick life", tweet)
 tweet = re.sub(r"yycweather", "Calgary Weather", tweet)
 tweet = re.sub(r"calgarysun", "Calgary Sun", tweet)
 tweet = re.sub(r"approachng", "approaching", tweet)
 tweet = re.sub(r"evng", "evening", tweet)
 tweet = re.sub(r"Sumthng", "something", tweet)
 tweet = re.sub(r"EllenPompeo", "Ellen Pompeo", tweet)
 tweet = re.sub(r"shondarhimes", "Shonda Rhimes", tweet)
 tweet = re.sub(r"ABCNetwork", "ABC Network", tweet)
 tweet = re.sub(r"SushmaSwaraj", "Sushma Swaraj", tweet)
 tweet = re.sub(r"pray4japan", "Pray for Japan", tweet)
 tweet = re.sub(r"hope4japan", "Hope for Japan", tweet)
 tweet = re.sub(r"Illusionimagess", "Illusion images", tweet)
 tweet = re.sub(r"SummerUnderTheStars", "Summer Under The Stars", tweet)
 tweet = re.sub(r"ShallWeDance", "Shall We Dance", tweet)
 tweet = re.sub(r"TCMParty", "TCM Party", tweet)
 tweet = re.sub(r"marijuananews", "marijuana news", tweet)
 tweet = re.sub(r"onbeingwithKristaTippett", "on being with Krista Tippett", tweet)
 tweet = re.sub(r"Beingtweets", "Being tweets", tweet)
 tweet = re.sub(r"newauthors", "new authors", tweet)
 tweet = re.sub(r"remedyyyy", "remedy", tweet)
 tweet = re.sub(r"44PM", "44 PM", tweet)
 tweet = re.sub(r"HeadlinesApp", "Headlines App", tweet)
 tweet = re.sub(r"40PM", "40 PM", tweet)
 tweet = re.sub(r"myswc", "Severe Weather Center", tweet)
 tweet = re.sub(r"ithats", "that is", tweet)
 tweet = re.sub(r"icouldsitinthismomentforever", "I could sit in this moment forever", tweet)
 tweet = re.sub(r"FatLoss", "Fat Loss", tweet)
 tweet = re.sub(r"02PM", "02 PM", tweet)
 tweet = re.sub(r"MetroFmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"Bstrd", "bastard", tweet)
 tweet = re.sub(r"bldy", "bloody", tweet)
 tweet = re.sub(r"MetrofmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"terrorismturn", "terrorism turn", tweet)
 tweet = re.sub(r"BBCNewsAsia", "BBC News Asia", tweet)
 tweet = re.sub(r"BehindTheScenes", "Behind The Scenes", tweet)
 tweet = re.sub(r"GeorgeTakei", "George Takei", tweet)
 tweet = re.sub(r"WomensWeeklyMag", "Womens Weekly Magazine", tweet)
 tweet = re.sub(r"SurvivorsGuidetoEarth", "Survivors Guide to Earth", tweet)
 tweet = re.sub(r"incubusband", "incubus band", tweet)
 tweet = re.sub(r"Babypicturethis", "Baby picture this", tweet)
 tweet = re.sub(r"BombEffects", "Bomb Effects", tweet)
 tweet = re.sub(r"win10", "Windows 10", tweet)
 tweet = re.sub(r"idkidk", "I do not know I do not know", tweet)
 tweet = re.sub(r"TheWalkingDead", "The Walking Dead", tweet)
 tweet = re.sub(r"amyschumer", "Amy Schumer", tweet)
 tweet = re.sub(r"crewlist", "crew list", tweet)
 tweet = re.sub(r"Erdogans", "Erdogan", tweet)
 tweet = re.sub(r"BBCLive", "BBC Live", tweet)
 tweet = re.sub(r"TonyAbbottMHR", "Tony Abbott", tweet)
 tweet = re.sub(r"paulmyerscough", "Paul Myerscough", tweet)
 tweet = re.sub(r"georgegallagher", "George Gallagher", tweet)
 tweet = re.sub(r"JimmieJohnson", "Jimmie Johnson", tweet)
 tweet = re.sub(r"pctool", "pc tool", tweet)
 tweet = re.sub(r"DoingHashtagsRight", "Doing Hashtags Right", tweet)
 tweet = re.sub(r"ThrowbackThursday", "Throwback Thursday", tweet)
 tweet = re.sub(r"SnowBackSunday", "Snowback Sunday", tweet)
 tweet = re.sub(r"LakeEffect", "Lake Effect", tweet)
 tweet = re.sub(r"RTphotographyUK", "Richard Thomas Photography UK", tweet)
 tweet = re.sub(r"BigBang_CBS", "Big Bang CBS", tweet)
 tweet = re.sub(r"writerslife", "writers life", tweet)
 tweet = re.sub(r"NaturalBirth", "Natural Birth", tweet)
 tweet = re.sub(r"UnusualWords", "Unusual Words", tweet)
 tweet = re.sub(r"wizkhalifa", "Wiz Khalifa", tweet)
 tweet = re.sub(r"acreativedc", "a creative DC", tweet)
 tweet = re.sub(r"vscodc", "vsco DC", tweet)
 tweet = re.sub(r"VSCOcam", "vsco camera", tweet)
 tweet = re.sub(r"TheBEACHDC", "The beach DC", tweet)
 tweet = re.sub(r"buildingmuseum", "building museum", tweet)
 tweet = re.sub(r"WorldOil", "World Oil", tweet)
 tweet = re.sub(r"redwedding", "red wedding", tweet)
 tweet = re.sub(r"AmazingRaceCanada", "Amazing Race Canada", tweet)
 tweet = re.sub(r"WakeUpAmerica", "Wake Up America", tweet)
 tweet = re.sub(r"\\Allahuakbar\", "Allahu Akbar", tweet)
 tweet = re.sub(r"bleased", "blessed", tweet)
 tweet = re.sub(r"nigeriantribune", "Nigerian Tribune", tweet)
 tweet = re.sub(r"HIDEO_KOJIMA_EN", "Hideo Kojima", tweet)
 tweet = re.sub(r"FusionFestival", "Fusion Festival", tweet)
 tweet = re.sub(r"50Mixed", "50 Mixed", tweet)
 tweet = re.sub(r"NoAgenda", "No Agenda", tweet)
 tweet = re.sub(r"WhiteGenocide", "White Genocide", tweet)
 tweet = re.sub(r"dirtylying", "dirty lying", tweet)
 tweet = re.sub(r"SyrianRefugees", "Syrian Refugees", tweet)
 tweet = re.sub(r"changetheworld", "change the world", tweet)
 tweet = re.sub(r"Ebolacase", "Ebola case", tweet)
 tweet = re.sub(r"mcgtech", "mcg technologies", tweet)
 tweet = re.sub(r"withweapons", "with weapons", tweet)
 tweet = re.sub(r"advancedwarfare", "advanced warfare", tweet)
 tweet = re.sub(r"letsFootball", "let us Football", tweet)
 tweet = re.sub(r"LateNiteMix", "late night mix", tweet)
 tweet = re.sub(r"PhilCollinsFeed", "Phil Collins", tweet)
 tweet = re.sub(r"RudyHavenstein", "Rudy Havenstein", tweet)
 tweet = re.sub(r"22PM", "22 PM", tweet)
 tweet = re.sub(r"54am", "54 AM", tweet)
 tweet = re.sub(r"38am", "38 AM", tweet)
 tweet = re.sub(r"OldFolkExplainStuff", "Old Folk Explain Stuff", tweet)
 tweet = re.sub(r"BlacklivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"InsaneLimits", "Insane Limits", tweet)
 tweet = re.sub(r"youcantsitwithus", "you cannot sit with us", tweet)
 tweet = re.sub(r"2k15", "2015", tweet)
 tweet = re.sub(r"TheIran", "Iran", tweet)
 tweet = re.sub(r"JimmyFallon", "Jimmy Fallon", tweet)
 tweet = re.sub(r"AlbertBrooks", "Albert Brooks", tweet)
 tweet = re.sub(r"defense_news", "defense news", tweet)
 tweet = re.sub(r"nuclearrcSA", "Nuclear Risk Control Self Assessment", tweet)
 tweet = re.sub(r"Auspol", "Australia Politics", tweet)
 tweet = re.sub(r"NuclearPower", "Nuclear Power", tweet)
 tweet = re.sub(r"WhiteTerrorism", "White Terrorism", tweet)
 tweet = re.sub(r"truthfrequencyradio", "Truth Frequency Radio", tweet)
 tweet = re.sub(r"ErasureIsNotEquality", "Erasure is not equality", tweet)
 tweet = re.sub(r"ProBonoNews", "Pro Bono News", tweet)
 tweet = re.sub(r"JakartaPost", "Jakarta Post", tweet)
 tweet = re.sub(r"toopainful", "too painful", tweet)
 tweet = re.sub(r"melindahaunton", "Melinda Haunton", tweet)
 tweet = re.sub(r"NoNukes", "No Nukes", tweet)
 tweet = re.sub(r"curryspcworld", "Currys PC World", tweet)
 tweet = re.sub(r"ineedcake", "I need cake", tweet)
 tweet = re.sub(r"blackforestgateau", "black forest gateau", tweet)
 tweet = re.sub(r"BBCOne", "BBC One", tweet)
 tweet = re.sub(r"AlexxPage", "Alex Page", tweet)
 tweet = re.sub(r"jonathanserrie", "Jonathan Serrie", tweet)
 tweet = re.sub(r"SocialJerkBlog", "Social Jerk Blog", tweet)
 tweet = re.sub(r"ChelseaVPeretti", "Chelsea Peretti", tweet)
 tweet = re.sub(r"irongiant", "iron giant", tweet)
 tweet = re.sub(r"RonFunches", "Ron Funches", tweet)
 tweet = re.sub(r"TimCook", "Tim Cook", tweet)
 tweet = re.sub(r"sebastianstanisaliveandwell", "Sebastian Stan is alive and well", tweet)
 tweet = re.sub(r"Madsummer", "Mad summer", tweet)
 tweet = re.sub(r"NowYouKnow", "Now you know", tweet)
 tweet = re.sub(r"concertphotography", "concert photography", tweet)
 tweet = re.sub(r"TomLandry", "Tom Landry", tweet)
 tweet = re.sub(r"showgirldayoff", "show girl day off", tweet)
 tweet = re.sub(r"Yougslavia", "Yugoslavia", tweet)
 tweet = re.sub(r"QuantumDataInformatics", "Quantum Data Informatics", tweet)
 tweet = re.sub(r"FromTheDesk", "From The Desk", tweet)
 tweet = re.sub(r"TheaterTrial", "Theater Trial", tweet)
 tweet = re.sub(r"CatoInstitute", "Cato Institute", tweet)
 tweet = re.sub(r"EmekaGift", "Emeka Gift", tweet)
 tweet = re.sub(r"LetsBe_Rational", "Let us be rational", tweet)
 tweet = re.sub(r"Cynicalreality", "Cynical reality", tweet)
 tweet = re.sub(r"FredOlsenCruise", "Fred Olsen Cruise", tweet)
 tweet = re.sub(r"NotSorry", "not sorry", tweet)
 tweet = re.sub(r"UseYourWords", "use your words", tweet)
 tweet = re.sub(r"WordoftheDay", "word of the day", tweet)
 tweet = re.sub(r"Dictionarycom", "Dictionary.com", tweet)
 tweet = re.sub(r"TheBrooklynLife", "The Brooklyn Life", tweet)
 tweet = re.sub(r"jokethey", "joke they", tweet)
 tweet = re.sub(r"nflweek1picks", "NFL week 1 picks", tweet)
 tweet = re.sub(r"uiseful", "useful", tweet)
 tweet = re.sub(r"JusticeDotOrg", "The American Association for Justice", tweet)
 tweet = re.sub(r"autoaccidents", "auto accidents", tweet)
 tweet = re.sub(r"SteveGursten", "Steve Gursten", tweet)
 tweet = re.sub(r"MichiganAutoLaw", "Michigan Auto Law", tweet)
 tweet = re.sub(r"birdgang", "bird gang", tweet)
 tweet = re.sub(r"nflnetwork", "NFL Network", tweet)
 tweet = re.sub(r"NYDNSports", "NY Daily News Sports", tweet)
 tweet = re.sub(r"RVacchianoNYDN", "Ralph Vacchiano NY Daily News", tweet)
 tweet = re.sub(r"EdmontonEsks", "Edmonton Eskimos", tweet)
 tweet = re.sub(r"david_brelsford", "David Brelsford", tweet)
 tweet = re.sub(r"TOI_India", "The Times of India", tweet)
 tweet = re.sub(r"hegot", "he got", tweet)
 tweet = re.sub(r"SkinsOn9", "Skins on 9", tweet)
 tweet = re.sub(r"sothathappened", "so that happened", tweet)
 tweet = re.sub(r"LCOutOfDoors", "LC Out Of Doors", tweet)
 tweet = re.sub(r"NationFirst", "Nation First", tweet)
 tweet = re.sub(r"IndiaToday", "India Today", tweet)
 tweet = re.sub(r"HLPS", "helps", tweet)
 tweet = re.sub(r"HOSTAGESTHROSW", "hostages throw", tweet)
 tweet = re.sub(r"SNCTIONS", "sanctions", tweet)
 tweet = re.sub(r"BidTime", "Bid Time", tweet)
 tweet = re.sub(r"crunchysensible", "crunchy sensible", tweet)
 tweet = re.sub(r"RandomActsOfRomance", "Random acts of romance", tweet)
 tweet = re.sub(r"MomentsAtHill", "Moments at hill", tweet)
 tweet = re.sub(r"eatshit", "eat shit", tweet)
 tweet = re.sub(r"liveleakfun", "live leak fun", tweet)
 tweet = re.sub(r"SahelNews", "Sahel News", tweet)
 tweet = re.sub(r"abc7newsbayarea", "ABC 7 News Bay Area", tweet)
 tweet = re.sub(r"facilitiesmanagement", "facilities management", tweet)
 tweet = re.sub(r"facilitydude", "facility dude", tweet)
 tweet = re.sub(r"CampLogistics", "Camp logistics", tweet)
 tweet = re.sub(r"alaskapublic", "Alaska public", tweet)
 tweet = re.sub(r"MarketResearch", "Market Research", tweet)
 tweet = re.sub(r"AccuracyEsports", "Accuracy Esports", tweet)
 tweet = re.sub(r"TheBodyShopAust", "The Body Shop Australia", tweet)
 tweet = re.sub(r"yychail", "Calgary hail", tweet)
 tweet = re.sub(r"yyctraffic", "Calgary traffic", tweet)
 tweet = re.sub(r"eliotschool", "eliot school", tweet)
 tweet = re.sub(r"TheBrokenCity", "The Broken City", tweet)
 tweet = re.sub(r"OldsFireDept", "Olds Fire Department", tweet)
 tweet = re.sub(r"RiverComplex", "River Complex", tweet)
 tweet = re.sub(r"fieldworksmells", "field work smells", tweet)
 tweet = re.sub(r"IranElection", "Iran Election", tweet)
 tweet = re.sub(r"glowng", "glowing", tweet)
 tweet = re.sub(r"kindlng", "kindling", tweet)
 tweet = re.sub(r"riggd", "rigged", tweet)
 tweet = re.sub(r"slownewsday", "slow news day", tweet)
 tweet = re.sub(r"MyanmarFlood", "Myanmar Flood", tweet)
 tweet = re.sub(r"abc7chicago", "ABC 7 Chicago", tweet)
 tweet = re.sub(r"copolitics", "Colorado Politics", tweet)
 tweet = re.sub(r"AdilGhumro", "Adil Ghumro", tweet)
 tweet = re.sub(r"netbots", "net bots", tweet)
 tweet = re.sub(r"byebyeroad", "bye bye road", tweet)
 tweet = re.sub(r"massiveflooding", "massive flooding", tweet)
 tweet = re.sub(r"EndofUS", "End of United States", tweet)
 tweet = re.sub(r"35PM", "35 PM", tweet)
 tweet = re.sub(r"greektheatrela", "Greek Theatre Los Angeles", tweet)
 tweet = re.sub(r"76mins", "76 minutes", tweet)
 tweet = re.sub(r"publicsafetyfirst", "public safety first", tweet)
 tweet = re.sub(r"livesmatter", "lives matter", tweet)
 tweet = re.sub(r"myhometown", "my hometown", tweet)
 tweet = re.sub(r"tankerfire", "tanker fire", tweet)
 tweet = re.sub(r"MEMORIALDAY", "memorial day", tweet)
 tweet = re.sub(r"MEMORIAL_DAY", "memorial day", tweet)
 tweet = re.sub(r"instaxbooty", "instagram booty", tweet)
 tweet = re.sub(r"Jerusalem_Post", "Jerusalem Post", tweet)
 tweet = re.sub(r"WayneRooney_INA", "Wayne Rooney", tweet)
 tweet = re.sub(r"VirtualReality", "Virtual Reality", tweet)
 tweet = re.sub(r"OculusRift", "Oculus Rift", tweet)
 tweet = re.sub(r"OwenJones84", "Owen Jones", tweet)
 tweet = re.sub(r"jeremycorbyn", "Jeremy Corbyn", tweet)
 tweet = re.sub(r"paulrogers002", "Paul Rogers", tweet)
 tweet = re.sub(r"mortalkombatx", "Mortal Kombat X", tweet)
 tweet = re.sub(r"mortalkombat", "Mortal Kombat", tweet)
 tweet = re.sub(r"FilipeCoelho92", "Filipe Coelho", tweet)
 tweet = re.sub(r"OnlyQuakeNews", "Only Quake News", tweet)
 tweet = re.sub(r"kostumes", "costumes", tweet)
 tweet = re.sub(r"YEEESSSS", "yes", tweet)
 tweet = re.sub(r"ToshikazuKatayama", "Toshikazu Katayama", tweet)
 tweet = re.sub(r"IntlDevelopment", "Intl Development", tweet)
 tweet = re.sub(r"ExtremeWeather", "Extreme Weather", tweet)
 tweet = re.sub(r"WereNotGruberVoters", "We are not gruber voters", tweet)
 tweet = re.sub(r"NewsThousands", "News Thousands", tweet)
 tweet = re.sub(r"EdmundAdamus", "Edmund Adamus", tweet)
 tweet = re.sub(r"EyewitnessWV", "Eye witness WV", tweet)
 tweet = re.sub(r"PhiladelphiaMuseu", "Philadelphia Museum", tweet)
 tweet = re.sub(r"DublinComicCon", "Dublin Comic Con", tweet)
 tweet = re.sub(r"NicholasBrendon", "Nicholas Brendon", tweet)
 tweet = re.sub(r"Alltheway80s", "All the way 80s", tweet)
 tweet = re.sub(r"FromTheField", "From the field", tweet)
 tweet = re.sub(r"NorthIowa", "North Iowa", tweet)
 tweet = re.sub(r"WillowFire", "Willow Fire", tweet)
 tweet = re.sub(r"MadRiverComplex", "Mad River Complex", tweet)
 tweet = re.sub(r"feelingmanly", "feeling manly", tweet)
 tweet = re.sub(r"stillnotoverit", "still not over it", tweet)
 tweet = re.sub(r"FortitudeValley", "Fortitude Valley", tweet)
 tweet = re.sub(r"CoastpowerlineTramTr", "Coast powerline", tweet)
 tweet = re.sub(r"ServicesGold", "Services Gold", tweet)
 tweet = re.sub(r"NewsbrokenEmergency", "News broken emergency", tweet)
 tweet = re.sub(r"Evaucation", "evacuation", tweet)
 tweet = re.sub(r"leaveevacuateexitbe", "leave evacuate exit be", tweet)
 tweet = re.sub(r"P_EOPLE", "PEOPLE", tweet)
 tweet = re.sub(r"Tubestrike", "tube strike", tweet)
 tweet = re.sub(r"CLASS_SICK", "CLASS SICK", tweet)
 tweet = re.sub(r"localplumber", "local plumber", tweet)
 tweet = re.sub(r"awesomejobsiri", "awesome job siri", tweet)
 tweet = re.sub(r"PayForItHow", "Pay for it how", tweet)
 tweet = re.sub(r"ThisIsAfrica", "This is Africa", tweet)
 tweet = re.sub(r"crimeairnetwork", "crime air network", tweet)
 tweet = re.sub(r"KimAcheson", "Kim Acheson", tweet)
 tweet = re.sub(r"cityofcalgary", "City of Calgary", tweet)
 tweet = re.sub(r"prosyndicate", "pro syndicate", tweet)
 tweet = re.sub(r"660NEWS", "660 NEWS", tweet)
 tweet = re.sub(r"BusInsMagazine", "Business Insurance Magazine", tweet)
 tweet = re.sub(r"wfocus", "focus", tweet)
 tweet = re.sub(r"ShastaDam", "Shasta Dam", tweet)
 tweet = re.sub(r"go2MarkFranco", "Mark Franco", tweet)
 tweet = re.sub(r"StephGHinojosa", "Steph Hinojosa", tweet)
 tweet = re.sub(r"Nashgrier", "Nash Grier", tweet)
 tweet = re.sub(r"NashNewVideo", "Nash new video", tweet)
 tweet = re.sub(r"IWouldntGetElectedBecause", "I would not get elected because", tweet)
 tweet = re.sub(r"SHGames", "Sledgehammer Games", tweet)
 tweet = re.sub(r"bedhair", "bed hair", tweet)
 tweet = re.sub(r"JoelHeyman", "Joel Heyman", tweet)
 tweet = re.sub(r"viaYouTube", "via YouTube", tweet)
 tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"mÌ¼sica", "music", tweet)
 tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
 tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
 tweet = re.sub(r"gawx", "Georgia Weather", tweet)
 tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
 tweet = re.sub(r"cawx", "California Weather", tweet)
 tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
 tweet = re.sub(r"azwx", "Arizona Weather", tweet)
 tweet = re.sub(r"alwx", "Alabama Weather", tweet)
 tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
 tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
 tweet = re.sub(r"Suruc", "Sanliurfa", tweet)
 return tweet<feature_engineering>
X_train, X_test, y_train, y_test = train_test_split(X, y_log, test_size=0.3, random_state=42 )
train['text'] = train['text'].apply(lambda x: clean(x))
test['text'] = test['text'].apply(lambda x: clean(x))<define_variables>
xStandardScaler = StandardScaler()
yStandardScaler = StandardScaler()
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
} 
<categorify>
X_train = xStandardScaler.fit_transform(X_train)
 X_test = xStandardScaler.transform(X_test )
def convert_abbrev(word):
return abbreviations[word.lower() ] if word.lower() in abbreviations.keys() else word<feature_engineering>
y_train = yStandardScaler.fit_transform(y_train.ravel().reshape(-1, 1))
y_test = yStandardScaler.transform(y_test.ravel().reshape(-1, 1))
def convert_abbrev_in_text(text):
text = text.lower()
tokens = word_tokenize(text)
 tokens = [convert_abbrev(word)for word in tokens]
text = ' '.join(tokens)
 return text
train['text'] = train['text'].apply(lambda x: convert_abbrev_in_text(x))
test['text'] = test['text'].apply(lambda x: convert_abbrev_in_text(x))<feature_engineering>
ModelCompList = list()
def remove_digit(text):
return re.sub('\w*\d+',' ',text)
 train['text'] = train['text'].apply(lambda text: remove_digit(text))
test['text'] = test['text'].apply(lambda text: remove_digit(text))<feature_engineering>
alphas = np.linspace(0,.1,num=21)
 lgLasso = LassoCV(cv=10,alphas=alphas)
 lgLasso.fit(X_train,y_train )
def extra_space(text):
return re.sub(r'\s+', ' ',text)
 train['text'] = train['text'].apply(lambda text: extra_space(text))
test['text'] = test['text'].apply(lambda text: extra_space(text))<feature_engineering>
print(lgLasso.alpha_ )
nlp = spacy.load('en_core_web_lg')
 def stop_word_lemma(text):
doc = nlp(text)
 text1 = ''
for token in doc:
if not token.is_stop:
text1 = text1+ ' '+ token.lemma_
return text1
train['text'] = train['text'].apply(lambda text: stop_word_lemma(text))
test['text'] = test['text'].apply(lambda text: stop_word_lemma(text))<feature_engineering>
predictionsLinear = lgLasso.predict(X_test )
cnt = Counter()
for text in train['text'].values:
for word in text.split() :
cnt[word]+=1
cnt.most_common(10 )<feature_engineering>
plt.scatter(np.exp(yStandardScaler.inverse_transform(y_test)) ,np.exp(yStandardScaler.inverse_transform(predictionsLinear)) )
FREQ_WORDS = set([w for(w,wc)in cnt.most_common(5)])
 def remove_freqwords(text):
return " ".join([word for word in text.split() if word not in FREQ_WORDS])
 train['text'] = train['text'].apply(lambda text: remove_freqwords(text))
test['text'] = test['text'].apply(lambda text: remove_freqwords(text))<feature_engineering>
sns.distplot(( yStandardScaler.inverse_transform(y_test)-yStandardScaler.inverse_transform(predictionsLinear)) ,bins=50);
n_rare_words = 10
RAREWORDS = set([w for(w,wc)in cnt.most_common() [:-n_rare_words-1: -1]])
 def remove_rarewords(text):
return " ".join([word for word in str(text ).split() if word not in RAREWORDS])
 train['text'] = train['text'].apply(lambda text: remove_rarewords(text))
test['text'] = test['text'].apply(lambda text: remove_rarewords(text))<split>
def evaluateModel(name,y_test,Predictions):
modelComp = dict()
modelComp['Name'] = name
modelComp['MAE'] = metrics.mean_absolute_error(np.exp(yStandardScaler.inverse_transform(y_test)) ,
np.exp(yStandardScaler.inverse_transform(Predictions)))
 modelComp['MSE'] = metrics.mean_squared_error(np.exp(yStandardScaler.inverse_transform(y_test)) ,
np.exp(yStandardScaler.inverse_transform(Predictions)))
 modelComp['RMSE'] = np.sqrt(metrics.mean_squared_error(np.exp(yStandardScaler.inverse_transform(y_test)) ,
np.exp(yStandardScaler.inverse_transform(Predictions))))
print('MAE:', modelComp['MAE'])
 print('MSE:', modelComp['MSE'])
 print('RMSE:', modelComp['RMSE'])
 ModelCompList.append(modelComp )
X = train.text
Y = train.target
X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size=0.15 )<choose_model_class>
evaluateModel('Lasso',y_test,predictionsLinear )
clf = Pipeline([("vectorizer", TfidfVectorizer(max_features = 10000,ngram_range=(1,2))),("classifier", OneVsRestClassifier(LogisticRegression() , n_jobs = 4)) ] )<train_model>
alphas = np.linspace(0,20,num=21)
 lgRidge = RidgeCV(cv=10,alphas=alphas)
 lgRidge.fit(X_train,y_train )
clf.fit(X_train, Y_train )<predict_on_test>
print(lgRidge.alpha_ )
y_pred = clf.predict(X_test)
 accuracy_score(Y_test,y_pred )<load_from_url>
predictionsLinear = lgRidge.predict(X_test )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
plt.scatter(np.exp(yStandardScaler.inverse_transform(y_test)) ,np.exp(yStandardScaler.inverse_transform(predictionsLinear)) )
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization 
<categorify>
sns.distplot(( yStandardScaler.inverse_transform(y_test)-yStandardScaler.inverse_transform(predictionsLinear)) ,bins=50);
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
evaluateModel('Ridge',y_test,predictionsLinear )
Dropout_num =0
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
if Dropout_num == 0:
out = Dense(1, activation='sigmoid' )(clf_output)
 else:
x = Dropout(Dropout_num )(clf_output)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model<import_modules>
alphas = np.linspace(0,10,num=21)
 l1_ratio = np.linspace(0,1,num=21)
 lgNet = ElasticNetCV(cv=10,alphas=alphas,l1_ratio=l1_ratio)
 lgNet.fit(X_train,y_train )
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Embedding, LSTM,Dense, SpatialDropout1D, Dropout
from keras.initializers import Constant
from keras.optimizers import Adam<choose_model_class>
print(lgNet.alpha_)
 print(lgNet.l1_ratio_ )
module_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1'
bert_layer = hub.KerasLayer(module_url, trainable=True )<feature_engineering>
predictionsLinear = lgNet.predict(X_test )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
plt.scatter(np.exp(yStandardScaler.inverse_transform(y_test)) ,np.exp(yStandardScaler.inverse_transform(predictionsLinear)) )
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values<choose_model_class>
sns.distplot(( yStandardScaler.inverse_transform(y_test)-yStandardScaler.inverse_transform(predictionsLinear)) ,bins=50);
learning_rate = 1e-5
model_BERT = build_model(bert_layer, max_len=160)
 model_BERT.summary()<train_model>
evaluateModel('Elastic Net',y_test,predictionsLinear )
checkpoint = ModelCheckpoint('model_BERT.h5', monitor='val_loss', save_best_only=True)
 train_history = model_BERT.fit(
 train_input, train_labels,
validation_split = 0.1,
epochs = 5,
callbacks=[checkpoint],
batch_size = 16
)<predict_on_test>
rfr = RandomForestRegressor(n_estimators=2500)
 parameters = {'min_samples_split': [2],
'max_depth': [8],
'min_samples_leaf' :[2],
'max_samples': [.7]}
model_BERT.load_weights('model_BERT.h5')
 test_pred_BERT = model_BERT.predict(test_input)
 test_pred_BERT_int = test_pred_BERT.round().astype('int' )<create_dataframe>
rfr_grid = GridSearchCV(rfr,
parameters,
cv = 2 )
pred = pd.DataFrame(test_pred_BERT, columns=['preds'])
 pred.plot.hist()<prepare_output>
rfr_grid.fit(X_train,y_train )
submission['target'] = test_pred_BERT_int
submission.head(10 )<save_to_csv>
print(rfr_grid.best_score_)
 print(rfr_grid.best_params_ )
submission.to_csv("submission.csv", index=False, header=True )<install_modules>
predictionsForest = rfr_grid.best_estimator_.predict(X_test )
!pip install /kaggle/input/tensorboard-220/tensorboard-2.2.0-py3-none-any.whl -q
!pip install /kaggle/input/pytorch-lightning/pytorch_lightning-0.9.0-py3-none-any.whl -q<set_options>
plt.scatter(yStandardScaler.inverse_transform(y_test),yStandardScaler.inverse_transform(predictionsForest))
%matplotlib inline 
<set_options>
evaluateModel('Random Forest',y_test,predictionsForest )
def set_seed(seed: int = 666)-> None:
np.random.seed(seed)
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 set_seed()<load_from_csv>
xgb = XGBRegressor()
parameters = {'objective':['reg:squarederror'],
'learning_rate': [.1],
'max_depth': [3],
'min_child_weight': [2],
'subsample': [0.7],
'colsample_bytree': [.7],
'colsample_bylevel':[.7],
'alpha' : [.05],
'lambda' : [.3],
'n_estimators': [2500]}
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 train_targets_nonscored = pd.read_csv('/kaggle/input/lish-moa/train_targets_nonscored.csv')
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv' )<count_values>
xgb_grid = GridSearchCV(xgb,
parameters,
cv = 2)
 xgb_grid.fit(X_train,
y_train )
train_features.cp_type.value_counts(normalize=True )<filter>
print(xgb_grid.best_score_)
 print(xgb_grid.best_params_ )
control_group = train_features.loc[train_features.cp_type == 'ctl_vehicle', 'sig_id']
train_targets_scored.loc[train_targets_scored['sig_id'].isin(control_group)].sum() [1:].sum()<count_values>
predictionsLinear = xgb_grid.predict(X_test )
test_features.cp_type.value_counts(normalize=True )<count_values>
plt.scatter(np.exp(yStandardScaler.inverse_transform(y_test)) ,np.exp(yStandardScaler.inverse_transform(predictionsLinear)) )
train_features.cp_time.value_counts(normalize=True )<count_values>
sns.distplot(( yStandardScaler.inverse_transform(y_test)-yStandardScaler.inverse_transform(predictionsLinear)) ,bins=50);
train_features.cp_dose.value_counts(normalize=True )<filter>
evaluateModel('XG Boost',y_test,predictionsLinear )
no_control_target = train_targets_scored.loc[~train_targets_scored['sig_id'].isin(control_group)]<sort_values>
xgb = LGBMRegressor()
parameters = {'learning_rate': [.1],
'max_depth': [6],
'min_child_weight': [1],
'subsample': [0.7],
'colsample_bytree': [.7],
'colsample_bylevel': [.7],
'reg_alpha' : [.1],
'reg_lambda' : [.3],
'n_estimators': [12500]}
plt.plot(sorted(train_features.loc[train_features['sig_id'] == 'id_79fb45fe7', [col for col in train_features if 'g-' in col]].values.reshape(-1, 1)))
 plt.title('sorted g- value of id_79fb45fe7');<create_dataframe>
xgb_grid = GridSearchCV(xgb,
parameters,
cv = 2)
 xgb_grid.fit(X_train,
y_train )
s = pd.DataFrame({'sig_id': test_features['sig_id'].values} )<feature_engineering>
print(xgb_grid.best_score_)
 print(xgb_grid.best_params_ )
s[train_targets_scored.columns[1:]] = 0.0001<filter>
predictionsLinear = xgb_grid.predict(X_test )
control_group = test_features.loc[test_features.cp_type == 'ctl_vehicle', 'sig_id']<feature_engineering>
plt.scatter(np.exp(yStandardScaler.inverse_transform(y_test)) ,np.exp(yStandardScaler.inverse_transform(predictionsLinear)) )
s.loc[s['sig_id'].isin(control_group), train_targets_scored.columns[1:]] = 0<save_to_csv>
sns.distplot(( yStandardScaler.inverse_transform(y_test)-yStandardScaler.inverse_transform(predictionsLinear)) ,bins=50);
s.to_csv('basic_submission.csv', index=False )<categorify>
evaluateModel('Light GBM',y_test,predictionsLinear )
train_features = pd.concat([train_features, pd.get_dummies(train_features['cp_time'], prefix='cp_time')], axis=1)
 train_features = pd.concat([train_features, pd.get_dummies(train_features['cp_dose'], prefix='cp_dose')], axis=1)
 train_features = pd.concat([train_features, pd.get_dummies(train_features['cp_type'], prefix='cp_type')], axis=1)
 train_features = train_features.drop(['cp_type', 'cp_time', 'cp_dose'], axis=1)
<create_dataframe>
X_train_nn, X_test_nn, y_train_nn, y_test_nn = train_test_split(X, y, test_size=0.3, random_state=42 )
class MoADataset(Dataset):
def __init__(
 self,
data,
targets = None,
targets1 = None,
mode = 'train' 
):

 self.mode = mode
self.data = data
self.targets = targets
self.targets1 = targets1
def __getitem__(self, idx: int)-> Dict[str, np.array]:
data = self.data[idx]
if self.targets is not None:
target = self.targets[idx]
target1 = self.targets1[idx]
else:
target = np.zeros(( 206,))
target1 = np.zeros(( 402,))
sample = {'data': torch.tensor(data ).float() ,
'target': torch.tensor(target ).float() ,
'target1': torch.tensor(target1 ).float() }
return sample
def __len__(self)-> int:
return len(self.data )<train_model>
print("X_train Shape: ",X_train_nn.shape)
 print("X_test Shape: ",X_test_nn.shape)
 print("y_train Shape: ",y_train_nn.shape)
 print("y_test Shape: ",y_test_nn.shape )
class MoADataModule(pl.LightningDataModule):
def __init__(self, hparams: Dict,
train_data, train_targets, train_targets1,
valid_data, valid_targets, valid_targets1):
super().__init__()
self.hparams = hparams
self.train_data = train_data
self.train_targets = train_targets
self.train_targets1 = train_targets1
self.valid_data = valid_data
self.valid_targets = valid_targets
self.valid_targets1 = valid_targets1
def prepare_data(self):
pass
def setup(self, stage=None):
self.train_dataset = MoADataset(data=self.train_data.iloc[:, 1:].values,
targets=self.train_targets.iloc[:, 1:].values,
targets1=self.train_targets1.iloc[:, 1:].values)
 self.valid_dataset = MoADataset(data=self.valid_data.iloc[:, 1:].values,
targets=self.valid_targets.iloc[:, 1:].values,
targets1=self.valid_targets1.iloc[:, 1:].values)
 def train_dataloader(self):
train_loader = torch.utils.data.DataLoader(
 self.train_dataset,
batch_size=1024,
num_workers=0,
shuffle=True, 
)
 return train_loader
def val_dataloader(self):
valid_loader = torch.utils.data.DataLoader(
 self.valid_dataset,
batch_size=1024,
num_workers=0,
shuffle=False, 
)
 return valid_loader
def test_dataloader(self):
return None 
<choose_model_class>
xStandardScaler = StandardScaler()
yStandardScaler = StandardScaler()
X_train_nn = xStandardScaler.fit_transform(X_train_nn)
 X_test_nn = xStandardScaler.transform(X_test_nn)
 y_train_nn = yStandardScaler.fit_transform(y_train_nn.ravel().reshape(-1, 1))
y_test_nn= yStandardScaler.transform(y_test_nn.ravel().reshape(-1, 1))
n_h_layers = 2048
learning_rate = 1e-3
criterion = nn.BCEWithLogitsLoss()
class Net(nn.Module):
def __init__(self, n_in, n_h, n_out, n_out1):
super(Net, self ).__init__()
self.fc1 = nn.Linear(n_in, n_h)
 self.fc2 = nn.Linear(n_h, math.ceil(n_h/4))
self.fc3 = nn.Linear(math.ceil(n_h/4), n_out)
 self.fc4 = nn.Linear(math.ceil(n_h/4), n_out1)
 self.bn = nn.BatchNorm1d(n_in)
 self.bn1 = nn.BatchNorm1d(n_h)
 self.bn2 = nn.BatchNorm1d(math.ceil(n_h/4))
self.drop = nn.Dropout(0.2)
 self.n_out = n_out
self.selu = nn.SELU()
self.sigm = nn.Sigmoid()
def forward(self, x, targets, targets1):
self.loss = criterion
x = self.fc1(self.bn(x))
x = self.selu(x)
 x = self.fc2(self.drop(self.bn1(x)))
 x = self.selu(x)
 x1 = self.fc3(self.bn2(x))
x2 = self.fc4(self.bn2(x))
loss =(self.loss(x1, targets)+ self.loss(x2, targets1)) / 2
real_loss = self.loss(x1, targets)
 out = self.sigm(x1)
 return out, loss, real_loss 
<choose_model_class>
model = Sequential()
model.add(Dense(256, input_dim=301))
model.add(LeakyReLU(alpha=.1))
model.add(Dropout (.1))
model.add(Dense(192, activation='relu'))
model.add(Dropout (.2))
model.add(Dense(128, activation='relu'))
model.add(Dropout (.1))
model.add(Dense(64, activation='relu'))
model.add(Dropout (.2))
model.add(Dense(32, activation='relu'))
model.add(Dropout (.1))
model.add(Dense(16, activation='relu'))
model.add(Dense(1))
model.summary()
class LitMoA(pl.LightningModule):
def __init__(self, hparams, model):
super(LitMoA, self ).__init__()
self.hparams = hparams
self.model = model
def forward(self, x, targets, targets1, *args, **kwargs):
return self.model(x, targets, targets1)
 def configure_optimizers(self):
optimizer = torch.optim.AdamW(self.model.parameters() , lr=learning_rate, weight_decay=0.001)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=2)
 return(
 [optimizer],
[{'scheduler': scheduler, 'interval': 'epoch', 'monitor': 'valid_loss'}], 
)
 def training_step(
 self, batch: torch.Tensor, batch_idx: int 
)-> Union[int, Dict[str, Union[torch.Tensor, Dict[str, torch.Tensor]]]]:
data = batch['data']
target = batch['target']
target1 = batch['target1']
out, loss, real_loss = self(data, target, target1)
 logs = {'train_loss': loss, 'real_train_loss': real_loss}
return {
'loss': loss, 'real_train_loss': real_loss,
'log': logs,
'progress_bar': logs
}
def training_epoch_end(self, outputs):
avg_loss = torch.stack([x['loss'] for x in outputs] ).mean()
real_avg_loss = torch.stack([x['real_train_loss'] for x in outputs] ).mean()
logs = {'train_loss': avg_loss, 'real_train_loss': real_avg_loss}
return {'log': logs, 'progress_bar': logs}
def validation_step(
 self, batch: torch.Tensor, batch_idx: int 
)-> Union[int, Dict[str, Union[torch.Tensor, Dict[str, torch.Tensor]]]]:
data = batch['data']
target = batch['target']
target1 = batch['target1']
out, loss, real_loss = self(data, target, target1)
 logs = {'valid_loss': loss, 'real_valid_loss': real_loss}
return {
'loss': loss, 'real_valid_loss': real_loss,
'log': logs,
'progress_bar': logs,
}
def validation_epoch_end(self, outputs):
avg_loss = torch.stack([x['loss'] for x in outputs] ).mean()
real_avg_loss = torch.stack([x['real_valid_loss'] for x in outputs] ).mean()
logs = {'valid_loss': avg_loss, 'real_valid_loss': real_avg_loss}
return {'valid_loss': avg_loss, 'log': logs, 'progress_bar': logs} 
<categorify>
model.compile(loss='mean_squared_error',optimizer='adam',metrics=['mse'] )
test_features = pd.concat([test_features, pd.get_dummies(test_features['cp_time'], prefix='cp_time')], axis=1)
 test_features = pd.concat([test_features, pd.get_dummies(test_features['cp_dose'], prefix='cp_dose')], axis=1)
 test_features = pd.concat([test_features, pd.get_dummies(test_features['cp_type'], prefix='cp_type')], axis=1)
 test_features = test_features.drop(['cp_type', 'cp_time', 'cp_dose'], axis=1)
 test_dataset = MoADataset(data=test_features.iloc[:, 1:].values)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=1024,
num_workers=0,
shuffle=False,
)<find_best_params>
history = model.fit(X_train_nn,y_train_nn,epochs=100,batch_size=32,callbacks=mcp,validation_data=(X_test_nn,y_test_nn))
class MetricsCallback(Callback):

 def __init__(self):
super().__init__()
self.metrics = []
def on_validation_end(self, trainer, pl_module):
self.metrics.append(trainer.callback_metrics )<define_search_space>
model = load_model('model.h5' )
hparams = {}<split>
predictionsNeural = model.predict(X_test_nn )
rkf = RepeatedKFold(n_splits=6, n_repeats=8, random_state=42)
 n = rkf.get_n_splits()
all_predictions = None
scores = []
for train_index, valid_index in rkf.split(train_features):
net = Net(n_in = 879, n_h = n_h_layers, n_out = 206, n_out1 = 402)
 train_data, valid_data = train_features.iloc[train_index, :], train_features.iloc[valid_index, :]
train_targets_scored_train, train_targets_scored_valid = train_targets_scored.iloc[train_index, :], train_targets_scored.iloc[valid_index, :]
train_targets_nonscored_train, train_targets_nonscored_valid = train_targets_nonscored.iloc[train_index, :], train_targets_nonscored.iloc[valid_index, :]
model = LitMoA(hparams=hparams, model=net)
 dm = MoADataModule(hparams=hparams,
train_data=train_data, train_targets=train_targets_scored_train, train_targets1=train_targets_nonscored_train,
valid_data=valid_data, valid_targets=train_targets_scored_valid, valid_targets1=train_targets_nonscored_valid)
 hparams = {}
metrics_callback = MetricsCallback()
trainer = pl.Trainer(
 early_stop_callback=EarlyStopping(monitor='valid_loss', patience=10, mode='min'),
checkpoint_callback=ModelCheckpoint(monitor='valid_loss', save_top_k=1, filepath='{epoch}_{valid_loss:.4f}', mode='min'),
gpus=1,
max_epochs=50,
log_save_interval=100,
num_sanity_val_steps=0,
gradient_clip_val=0.5,
weights_summary='full',
callbacks = [metrics_callback] 
)
 trainer.fit(model, dm)
 score = metrics_callback.metrics[-1]['real_valid_loss'].item()
scores.append(score)
 predictions = np.zeros(( test_features.shape[0], 206))
model_inference = model.model
model_inference.eval()
for ind, batch in enumerate(test_loader):
p = model_inference(batch['data'], batch['target'], batch['target1'])[0].detach().cpu().numpy()
predictions[ind * 1024:(ind + 1)* 1024] = p
if all_predictions is None:
all_predictions = predictions
else:
all_predictions += predictions
all_predictions = all_predictions / n<compute_test_metric>
modelComp = dict()
modelComp['Name'] = 'Neural Network'
modelComp['MAE'] = metrics.mean_absolute_error(yStandardScaler.inverse_transform(y_test_nn),
yStandardScaler.inverse_transform(predictionsNeural))
modelComp['MSE'] = metrics.mean_squared_error(yStandardScaler.inverse_transform(y_test_nn),
yStandardScaler.inverse_transform(predictionsNeural))
modelComp['RMSE'] = np.sqrt(metrics.mean_squared_error(yStandardScaler.inverse_transform(y_test_nn),
yStandardScaler.inverse_transform(predictionsNeural)))
 print('MAE:', modelComp['MAE'])
 print('MSE:', modelComp['MSE'])
 print('RMSE:', modelComp['RMSE'])
 ModelCompList.append(modelComp )
print(f'Mean score: {np.mean(scores):.4f}.Std score: {np.std(scores):.4f}' )<create_dataframe>
df_models = pd.DataFrame(ModelCompList)
 df_models.head()
s = pd.DataFrame({'sig_id': test_features['sig_id'].values} )<feature_engineering>
df_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 df_test.head()
for col in train_targets_scored.columns[1:].tolist() :
s[col] = 0<feature_engineering>
IdTest = df_test['Id']
df_test.drop('Id',axis=1,inplace=True )
s.loc[:, train_targets_scored.columns[1:]] = all_predictions<load_from_csv>
df_test.loc[(df_test['MSZoning'].isnull())&(df_test['MSSubClass'].astype(int)<70),'MSZoning'] = 'RL'
df_test.loc[(df_test['MSZoning'].isnull())&(df_test['MSSubClass'].astype(int)>=70),'MSZoning'] = 'RM'
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 s.loc[s['sig_id'].isin(test_features.loc[test_features['cp_type'] =='ctl_vehicle', 'sig_id']), train_targets_scored.columns[1:]] = 0<save_to_csv>
df_test.loc[df_test['Utilities'].isnull() ,'Utilities'] = 'AllPub'
s.to_csv('submission.csv', index=False )<import_modules>
df_test.loc[df_test['Exterior1st'].isnull() ,'Exterior1st'] = 'Wd Sdng'
import torch
import torch.nn as nn
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt<load_from_csv>
df_test.loc[df_test['Exterior2nd'].isnull() ,'Exterior2nd'] = 'Wd Sdng'
data = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 target = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 del data['sig_id']
categorical_cols = ['cp_type', 'cp_dose']
categorical_features = data[categorical_cols]
data.drop(categorical_cols, axis=1, inplace=True)
 categorical_features = OneHotEncoder(sparse=False ).fit_transform(categorical_features)
 data = StandardScaler().fit_transform(data)
 X = np.concatenate(( categorical_features, data), axis=1)
 y = target.to_numpy() [:,1:].astype(float )<define_variables>
df_test.loc[df_test['BsmtFinSF1'].isnull() ,'BsmtFinSF1'] = 180
def k_folds_gen(*arrs, test_size=0.1):
indeces = np.random.permutation(arrs[0].shape[0])
 batch_size = int(arrs[0].shape[0] * 0.1)
 for start in range(0, len(indeces), batch_size):
indx_val = indeces[start: start+batch_size]
indx_train = np.concatenate(( indeces[:start], indeces[start+batch_size:]), axis=0)
 to_return = []
for arr in arrs:
to_return.append(arr[indx_train])
 to_return.append(arr[indx_val])
 yield tuple(to_return )<define_variables>
df_test.loc[df_test['BsmtFinSF2'].isnull() ,'BsmtFinSF2'] = 374
def batch_iterator(*arrs, batch_size):
indeces = np.random.permutation(arrs[0].shape[0])
 for start in range(0, len(indeces), batch_size):
indx = indeces[start: start+batch_size]
to_return = []
for arr in arrs:
to_return.append(arr[indx])
 yield tuple(to_return )<choose_model_class>
df_test.loc[df_test['BsmtUnfSF'].isnull() ,'BsmtUnfSF'] = 340
def create_N_models(N, device):
models = [None] * N
for i in range(len(models)) :
models[i] = nn.Sequential(
 nn.Linear(877, 256),
nn.BatchNorm1d(256),
nn.Dropout(p=0.8),
nn.LeakyReLU() ,
nn.Linear(256, 206),
nn.Sigmoid() , 
)
 models[i].to(device)
 return models<load_pretrained>
df_test.loc[df_test['TotalBsmtSF'].isnull() ,'TotalBsmtSF'] = 894
device = "cuda" if torch.cuda.is_available() else "cpu"
models = create_N_models(10, device)
 for i in range(len(models)) :
models[i].load_state_dict(torch.load(f"/kaggle/input/k-folds-weights/model_{i}.pth", map_location=device))
batch_size=128<load_from_csv>
df_test.loc[(df_test['BsmtFullBath'].isnull())&(df_test['FullBath']==1),'BsmtFullBath'] = 0
df_test.loc[(df_test['BsmtFullBath'].isnull())&(df_test['FullBath']==3),'BsmtFullBath'] = 1
test_data = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 sig_id = test_data['sig_id']
del test_data['sig_id']
categorical_features = test_data[categorical_cols]
test_data.drop(categorical_cols, axis=1, inplace=True)
 categorical_features = OneHotEncoder(sparse=False ).fit_transform(categorical_features)
 test_data = StandardScaler().fit_transform(test_data)
 X_test = np.concatenate(( categorical_features, test_data), axis=1 )<predict_on_test>
df_test.loc[df_test['BsmtHalfBath'].isnull() ,'BsmtHalfBath'] = 0
for i in range(len(models)) :
models[i].eval()
with torch.no_grad() :
predictions =(sum(map(lambda x: x(torch.tensor(X_test, dtype=torch.float32 ).to(device)) , models)) /
len(models)).to('cpu' ).detach().numpy()
predictions.shape<prepare_output>
df_test.loc[df_test['KitchenQual'].isnull() ,'KitchenQual'] = 'TA'
submission = pd.DataFrame(predictions)
 submission.columns = target.columns[1:]
submission['sig_id'] = sig_id
submission.head()<save_to_csv>
df_test.loc[(df_test['Functional'].isnull())&(df_train['ScreenPorch']<1), 'Functional'] = 'Typ'
df_test.loc[(df_test['Functional'].isnull())&(df_train['ScreenPorch']>1), 'Functional'] = 'Mod'
submission.to_csv('submission.csv', index=False )<save_to_csv>
df_test.loc[df_test['GarageCars'].isnull() ,'GarageCars']= 2
submission.to_csv('submission.csv', index=False )<install_modules>
df_test.loc[df_test['GarageArea'].isnull() ,'GarageArea']= 324
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet<import_modules>
df_test.loc[df_test['SaleType'].isnull() ,'SaleType'] = 'WD'
import os
import pickle
import joblib
import numpy as np
import pandas as pd
import tensorflow as tf
import torch
import tqdm
from gauss_rank_scaler import GaussRankScaler
from pytorch_tabnet.tab_model import TabNetRegressor<feature_engineering>
df_test['Lat'] = df_test['Neighborhood'].map(lambda neighbor: dict_neighbor[neighbor]['lat'])
 df_test['Lon'] = df_test['Neighborhood'].map(lambda neighbor: dict_neighbor[neighbor]['lon'] )
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "true"<create_dataframe>
Categorical_features_test = df_test.select_dtypes(include=['object'])
 Numerical_features_test = df_test.select_dtypes(exclude=['object'] )
def compute_row_statistics(X, prefix=""):
Xt = pd.DataFrame()
for agg_func in [
"mean",
"std",
"kurtosis",
"skew",
]:
Xt[f"{prefix}{agg_func}"] = X.agg(agg_func, axis=1)
 return Xt<train_model>
Categorical_Encoded_test = ohe.transform(Categorical_features_test)
 Categorical_Encoded_Frame_test = pd.DataFrame(Categorical_Encoded_test, columns= ohe.get_feature_names(Categorical_features_test.columns))
Categorical_Encoded_Frame_test.head()
class ClippedFeatures(BaseEstimator, TransformerMixin):
def __init__(self, copy=True, high=0.99, low=0.01):
self.copy = copy
self.high = high
self.low = low
def fit(self, X, y=None):
self.data_max_ = X.quantile(q=self.high)
 self.data_min_ = X.quantile(q=self.low)
 return self
def transform(self, X):
if self.copy:
X = X.copy()
X.clip(self.data_min_, self.data_max_, axis=1, inplace=True)
 return X<init_hyperparams>
LotFrontageTestX = Numerical_features_test['LotFrontage']
GarageYrBltTestX = Numerical_features_test['GarageYrBlt']
Numerical_features_test.drop(['LotFrontage','GarageYrBlt'],axis=1,inplace=True )
class DynamicNet(object):
def __init__(self, c0, lr):
self.models = []
self.c0 = c0
self.lr = lr
self.boost_rate = nn.Parameter(
 torch.tensor(lr, requires_grad=True, device=device)
)
 def add(self, model):
self.models.append(model)
 def parameters(self):
params = []
for m in self.models:
params.extend(m.parameters())
 params.append(self.boost_rate)
 return params
def zero_grad(self):
for m in self.models:
m.zero_grad()
def to_cuda(self):
for m in self.models:
m.cuda()
def to_eval(self):
for m in self.models:
m.eval()
def to_train(self):
for m in self.models:
m.train(True)
 def forward(self, x):
if len(self.models)== 0:
batch = x.shape[0]
c0 = np.repeat(self.c0.detach().cpu().numpy().reshape(1, -1), batch, axis=0)
 c0 = torch.Tensor(c0 ).cuda() if device == "cuda" else torch.Tensor(c0)
 return None, c0
middle_feat_cum = None
prediction = None
with torch.no_grad() :
for m in self.models:
if middle_feat_cum is None:
middle_feat_cum, prediction = m(x, middle_feat_cum)
 else:
middle_feat_cum, pred = m(x, middle_feat_cum)
 prediction += pred
return middle_feat_cum, self.c0 + self.boost_rate * prediction
def forward_grad(self, x):
if len(self.models)== 0:
batch = x.shape[0]
c0 = np.repeat(self.c0.detach().cpu().numpy().reshape(1, -1), batch, axis=0)
 return None, torch.Tensor(c0 ).cuda()
middle_feat_cum = None
prediction = None
for m in self.models:
if middle_feat_cum is None:
middle_feat_cum, prediction = m(x, middle_feat_cum)
 else:
middle_feat_cum, pred = m(x, middle_feat_cum)
 prediction += pred
return middle_feat_cum, self.c0 + self.boost_rate * prediction
@classmethod
def from_file(cls, path, builder, device):
d = torch.load(path, map_location=device)
 net = DynamicNet(d["c0"], d["lr"])
 net.boost_rate = d["boost_rate"]
for stage, m in enumerate(d["models"]):
submod = builder(stage)
 submod.load_state_dict(m)
 net.add(submod)
 return net
def to_file(self, path):
models = [m.state_dict() for m in self.models]
d = {
"models": models,
"c0": self.c0,
"lr": self.lr,
"boost_rate": self.boost_rate,
}
torch.save(d, path)
 def predict(self, X, device, batch_size=32):
self.to_eval()
for m in self.models:
m.to(device)
 dataset = TestDataset(X)
 data_loader = DataLoader(dataset, batch_size=batch_size)
 predictions = []
with torch.no_grad() :
for data in data_loader:
x = data["x"].to(device)
 _, pred = self.forward(x)
 predictions.append(pred.sigmoid().detach().cpu().numpy())
 return np.concatenate(predictions)
 class MLP_2HL(nn.Module):
def __init__(self, dim_in, dim_hidden, dim_out, sparse=False, bn=True):
super().__init__()
self.bn2 = nn.BatchNorm1d(dim_in)
 self.layer1 = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(dim_in, dim_hidden),
nn.ReLU() ,
nn.BatchNorm1d(dim_hidden),
nn.Dropout(0.4),
nn.Linear(dim_hidden, dim_hidden), 
)
 self.layer2 = nn.Sequential(
 nn.ReLU() ,
nn.Linear(dim_hidden, dim_out), 
)
 def forward(self, x, lower_f):
if lower_f is not None:
x = torch.cat([x, lower_f], dim=1)
 x = self.bn2(x)
 middle_feat = self.layer1(x)
 out = self.layer2(middle_feat)
 return middle_feat, out
@classmethod
def get_model(cls, stage, dim_in, dim_hidden, dim_out):
if stage != 0:
dim_in += dim_hidden
return MLP_2HL(dim_in, dim_hidden, dim_out)
 class TestDataset(object):
def __init__(self, X):
self.X = np.asarray(X)
 def __len__(self):
return self.X.shape[0]
def __getitem__(self, i):
return {"x": torch.tensor(self.X[i], dtype=torch.float)}<compute_test_metric>
X_test_Full = Categorical_Encoded_Frame_test.join(Numerical_features_test ).join(LotFrontageTestX ).join(GarageYrBltTestX)
 X_testcolumns = X_test_Full.columns
X_test_Full.head()
class LogitsLogLoss(Metric):
def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):
logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-15)+ y_true * np.log(
 logits + 1e-15 
)
 return -np.mean(aux )<predict_on_test>
X_test_Full = KNNImputer(n_neighbors=5 ).fit_transform(X_test_Full )
def predict_with_pytorch_model(model, X, device, batch_size=32):
model.eval()
dataset = TestDataset(X)
 data_loader = DataLoader(dataset, batch_size=batch_size)
 predictions = []
with torch.no_grad() :
for data in data_loader:
inputs = data["x"].to(device)
 outputs = model(inputs)
 predictions.append(outputs.sigmoid().detach().cpu().numpy())
 return np.concatenate(predictions)
 class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size, rate=0.2619422201258426):
super().__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(rate)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(rate)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<init_hyperparams>
X_test_Frame = pd.DataFrame(X_test_Full,columns=X_testcolumns)
 X_test_Frame.head()
class StackedTabNetClassifier(tf.keras.Model):
def __init__(
 self,
num_classes,
batch_momentum=0.98,
epsilon=1e-05,
feature_columns=None,
feature_dim=64,
norm_type="group",
num_decision_steps=5,
num_features=None,
num_groups=2,
num_layers=1,
output_dim=64,
relaxation_factor=1.5,
sparsity_coefficient=1e-05,
virtual_batch_size=None,
**kwargs 
):
super().__init__(**kwargs)
 self.stacked_tabnet = StackedTabNet(
 feature_columns,
batch_momentum=batch_momentum,
epsilon=epsilon,
feature_dim=feature_dim,
norm_type=norm_type,
num_decision_steps=num_decision_steps,
num_features=num_features,
num_groups=num_groups,
num_layers=num_layers,
output_dim=output_dim,
relaxation_factor=relaxation_factor,
sparsity_coefficient=sparsity_coefficient,
virtual_batch_size=virtual_batch_size, 
)
 self.classifier = tf.keras.layers.Dense(
 num_classes, activation="sigmoid", use_bias=False 
)
 def call(self, inputs, training=None):
x = self.stacked_tabnet(inputs, training=training)
 return self.classifier(x )<init_hyperparams>
Xtest = StandardScaler()
Ytest = StandardScaler()
X_Full = Xtest.fit_transform(X)
 y_Full = Ytest.fit_transform(y_log.ravel().reshape(-1,1))
class TabNetClassifier(tf.keras.Model):
def __init__(
 self,
num_classes,
batch_momentum=0.98,
epsilon=1e-05,
feature_columns=None,
feature_dim=64,
norm_type="group",
num_decision_steps=5,
num_features=None,
num_groups=1,
output_dim=64,
relaxation_factor=1.5,
sparsity_coefficient=1e-05,
virtual_batch_size=None,
**kwargs 
):
super().__init__(**kwargs)
 self.tabnet = TabNet(
 feature_columns,
batch_momentum=batch_momentum,
epsilon=epsilon,
feature_dim=feature_dim,
norm_type=norm_type,
num_decision_steps=num_decision_steps,
num_features=num_features,
num_groups=num_groups,
output_dim=output_dim,
relaxation_factor=relaxation_factor,
sparsity_coefficient=sparsity_coefficient,
virtual_batch_size=virtual_batch_size,
**kwargs 
)
 self.classifier = tf.keras.layers.Dense(
 num_classes, activation="sigmoid", use_bias=False 
)
 def call(self, inputs, training=None):
x = self.tabnet(inputs, training=training)
 return self.classifier(x )<compute_train_metric>
lgLassoFinal = LassoCV(cv=5,alphas =[.02])
 lgLassoFinal.fit(X_Full,y_Full )
def gelu(x):
cdf = 0.5 *(1.0 + tf.tanh(( np.sqrt(2 / np.pi)*(x + 0.044715 * tf.pow(x, 3)))))
 return x * cdf
def point_wise_feed_forward_network(d_model, dff):
return tf.keras.Sequential(
 [
tf.keras.layers.Dense(dff, activation=gelu),
tf.keras.layers.Dense(d_model),
] 
)
 def scaled_dot_product_attention(q, k, v, mask):
matmul_qk = tf.matmul(q, k, transpose_b=True)
 dk = tf.cast(tf.shape(k)[-1], tf.float32)
 scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
 if mask is not None:
scaled_attention_logits += mask * -1e09
attention_weights = tf.nn.softmax(
 scaled_attention_logits, axis=-1 
)
 output = tf.matmul(attention_weights, v)
 return output, attention_weights
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.1):
super().__init__()
self.mha = MultiHeadAttention(d_model, num_heads)
 self.ffn = point_wise_feed_forward_network(d_model, dff)
 self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-06)
 self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-06)
 self.dropout1 = tf.keras.layers.Dropout(rate)
 self.dropout2 = tf.keras.layers.Dropout(rate)
 def call(self, x, training, mask):
attn_output, _ = self.mha(x, x, x, mask)
 attn_output = self.dropout1(attn_output, training=training)
 out1 = self.layernorm1(x + attn_output)
 ffn_output = self.ffn(out1)
 ffn_output = self.dropout2(ffn_output, training=training)
 out2 = self.layernorm2(
 out1 + ffn_output 
)
 return out2
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads):
super().__init__()
self.num_heads = num_heads
self.d_model = d_model
self.depth = d_model // self.num_heads
self.wq = tf.keras.layers.Dense(d_model)
 self.wk = tf.keras.layers.Dense(d_model)
 self.wv = tf.keras.layers.Dense(d_model)
 self.dense = tf.keras.layers.Dense(d_model)
 def split_heads(self, x, batch_size):
x = tf.reshape(x,(batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
 def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.wq(q)
 k = self.wk(k)
 v = self.wv(v)
 q = self.split_heads(q, batch_size)
 k = self.split_heads(k, batch_size)
 v = self.split_heads(v, batch_size)
 scaled_attention, attention_weights = scaled_dot_product_attention(
 q, k, v, mask 
)
 scaled_attention = tf.transpose(
 scaled_attention, perm=[0, 2, 1, 3] 
)
 concat_attention = tf.reshape(
 scaled_attention,(batch_size, -1, self.d_model)
)
 output = self.dense(concat_attention)
 return output, attention_weights
class TransformerEncoder(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff, rate=0.1, **kwargs):
super().__init__(**kwargs)
 self.d_model = d_model
self.num_layers = num_layers
self.num_heads = num_heads
self.dff = dff
self.rate = rate
self.embedding = tfa.layers.WeightNormalization(
 tf.keras.layers.Dense(self.d_model)
)
 self.enc_layers = [
EncoderLayer(self.d_model, self.num_heads, self.dff, self.rate)
 for _ in range(self.num_layers)
 ]
self.dropout = tf.keras.layers.Dropout(self.rate)
 def get_config(self):
config = super().get_config().copy()
config.update(
 {
"d_model": self.d_model,
"dff": self.dff,
"num_layers": self.num_layers,
"num_heads": self.num_heads,
"rate": self.rate,
} 
)
 return config
def call(self, x, training, mask=None):
seq_len = tf.shape(x)[1]
x = self.embedding(x)
 x = self.dropout(x, training=training)
 for i in range(self.num_layers):
x = self.enc_layers[i](x, training, mask)
 return x
class TransformerEncoderByZhang(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff, rate=0.1, **kwargs):
super().__init__()
self.d_model = d_model
self.num_layers = num_layers
self.num_heads = num_heads
self.dff = dff
self.rate = rate
self.embedding = tf.keras.layers.Dense(self.d_model)
 self.enc_layers = [
EncoderLayer(self.d_model, self.num_heads, self.dff, self.rate)
 for _ in range(self.num_layers)
 ]
self.dropout = tf.keras.layers.Dropout(self.rate)
 def get_config(self):
config = super().get_config().copy()
config.update(
 {
"d_model": self.d_model,
"dff": self.dff,
"num_layers": self.num_layers,
"num_heads": self.num_heads,
"rate": self.rate,
} 
)
 return config
def call(self, x, training, mask=None):
seq_len = tf.shape(x)[1]
x = self.embedding(x)
 x = self.dropout(x, training=training)
 for i in range(self.num_layers):
x = self.enc_layers[i](x, training, mask)
 return x<load_pretrained>
X_test_Frame = Xtest.transform(X_test_Frame )
with open(".. /input/preprocessor-fit/clipped_features.pkl", "rb")as f:
clipped_features = pickle.load(f)
 with open(".. /input/preprocessor-fit/counts.pkl", "rb")as f:
counts = pickle.load(f)
 with open(".. /input/mlp-for-ensemble/kaggle_upload/stacked_tabnet_params.pkl", "rb")as f:
params_stacked_tabnet = pickle.load(f)
 with open(".. /input/tabnetclassifier-fit/params.pkl", "rb")as f:
params_tabnet = pickle.load(f)
 with open(".. /input/votingclassifier-fit-without-lightgm-and-svm/weights.pkl", "rb")as f:
weights = pickle.load(f)
 with open(".. /input/pytorch-mlp-tabnet-many-fe-train/rank_gauss.pkl", "rb")as f:
scalers = pickle.load(f)
 with open(".. /input/pytorch-mlp-tabnet-many-fe-train/gen_pca.pkl", "rb")as f:
pca_g = pickle.load(f)
 with open(".. /input/pytorch-mlp-tabnet-many-fe-train/cel_pca.pkl", "rb")as f:
pca_c = pickle.load(f)
 with open(".. /input/pytorch-mlp-tabnet-many-fe-train/var_thresh.pkl", "rb")as f:
selector = pickle.load(f)
<load_from_csv>
ypredLasso = lgLassoFinal.predict(X_test_Frame )
dtype = {"cp_type": "category", "cp_dose": "category"}
index_col = "sig_id"
Y = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv", index_col=index_col)
 test_features = pd.read_csv(
 ".. /input/lish-moa/test_features.csv", dtype=dtype, index_col=index_col 
)
 columns = Y.columns
index = test_features.index
c_prefix = "c-"
g_prefix = "g-"
c_columns = test_features.columns.str.startswith(c_prefix)
 c_columns = list(test_features.columns[c_columns])
 g_columns = test_features.columns.str.startswith(g_prefix)
 g_columns = list(test_features.columns[g_columns] )<statistical_test>
lgRidgeFinal = RidgeCV(cv=5,alphas =[20])
 lgRidgeFinal.fit(X_Full,y_Full )
X_test = test_features.select_dtypes("number" ).copy()
X_test_stats_c = compute_row_statistics(X_test.loc[:, c_columns], prefix=c_prefix)
 X_test_stats_g = compute_row_statistics(X_test.loc[:, g_columns], prefix=g_prefix)
 X_test_clipped = clipped_features.transform(X_test)
 X_test = pd.concat([X_test_clipped, X_test_stats_c, X_test_stats_g], axis=1 )<normalization>
ypredRidge = lgRidgeFinal.predict(X_test_Frame )
X_test_hirune924 = test_features[g_columns + c_columns].copy()
for column in g_columns + c_columns:
X_test_hirune924[column] = scalers[column].transform(X_test_hirune924[[column]])
 X_test_pca_g = pca_g.transform(X_test_hirune924[g_columns])
 X_test_pca_c = pca_c.transform(X_test_hirune924[c_columns])
 X_test_hirune924 = np.concatenate(
 [X_test_hirune924, X_test_pca_g, X_test_pca_c], axis=1 
)
 X_test_hirune924 = selector.transform(X_test_hirune924)
 X_test_hirune924 = pd.DataFrame(X_test_hirune924, index=index)
 X_test_hirune924 = pd.concat(
 [test_features[["cp_time", "cp_dose"]], X_test_hirune924], axis=1 
)
 X_test_hirune924 = pd.get_dummies(X_test_hirune924, columns=["cp_time", "cp_dose"] )<define_variables>
xgbFinal = XGBRegressor(objective='reg:squarederror',
learning_rate=.1,
max_depth= 3,
min_child_weight= 2,
subsample= 0.7,
colsample_bytree=.7,
colsample_bylevel=.7,
alpha =.05,
reg_lambda =.3,
n_estimators= 2500 )
selected_columns = [
"g-0",
"g-7",
"g-8",
"g-10",
"g-13",
"g-17",
"g-20",
"g-22",
"g-24",
"g-26",
"g-28",
"g-29",
"g-30",
"g-31",
"g-32",
"g-34",
"g-35",
"g-36",
"g-37",
"g-38",
"g-39",
"g-41",
"g-46",
"g-48",
"g-50",
"g-51",
"g-52",
"g-55",
"g-58",
"g-59",
"g-61",
"g-62",
"g-63",
"g-65",
"g-66",
"g-67",
"g-68",
"g-70",
"g-72",
"g-74",
"g-75",
"g-79",
"g-83",
"g-84",
"g-85",
"g-86",
"g-90",
"g-91",
"g-94",
"g-95",
"g-96",
"g-97",
"g-98",
"g-100",
"g-102",
"g-105",
"g-106",
"g-112",
"g-113",
"g-114",
"g-116",
"g-121",
"g-123",
"g-126",
"g-128",
"g-131",
"g-132",
"g-134",
"g-135",
"g-138",
"g-139",
"g-140",
"g-142",
"g-144",
"g-145",
"g-146",
"g-147",
"g-148",
"g-152",
"g-155",
"g-157",
"g-158",
"g-160",
"g-163",
"g-164",
"g-165",
"g-170",
"g-173",
"g-174",
"g-175",
"g-177",
"g-178",
"g-181",
"g-183",
"g-185",
"g-186",
"g-189",
"g-192",
"g-194",
"g-195",
"g-196",
"g-197",
"g-199",
"g-201",
"g-202",
"g-206",
"g-208",
"g-210",
"g-213",
"g-214",
"g-215",
"g-220",
"g-226",
"g-228",
"g-229",
"g-235",
"g-238",
"g-241",
"g-242",
"g-243",
"g-244",
"g-245",
"g-248",
"g-250",
"g-251",
"g-254",
"g-257",
"g-259",
"g-261",
"g-266",
"g-270",
"g-271",
"g-272",
"g-275",
"g-278",
"g-282",
"g-287",
"g-288",
"g-289",
"g-291",
"g-293",
"g-294",
"g-297",
"g-298",
"g-301",
"g-303",
"g-304",
"g-306",
"g-308",
"g-309",
"g-310",
"g-311",
"g-314",
"g-315",
"g-316",
"g-317",
"g-320",
"g-321",
"g-322",
"g-327",
"g-328",
"g-329",
"g-332",
"g-334",
"g-335",
"g-336",
"g-337",
"g-339",
"g-342",
"g-344",
"g-349",
"g-350",
"g-351",
"g-353",
"g-354",
"g-355",
"g-357",
"g-359",
"g-360",
"g-364",
"g-365",
"g-366",
"g-367",
"g-368",
"g-369",
"g-374",
"g-375",
"g-377",
"g-379",
"g-385",
"g-386",
"g-390",
"g-392",
"g-393",
"g-400",
"g-402",
"g-406",
"g-407",
"g-409",
"g-410",
"g-411",
"g-414",
"g-417",
"g-418",
"g-421",
"g-423",
"g-424",
"g-427",
"g-429",
"g-431",
"g-432",
"g-433",
"g-434",
"g-437",
"g-439",
"g-440",
"g-443",
"g-449",
"g-458",
"g-459",
"g-460",
"g-461",
"g-464",
"g-467",
"g-468",
"g-470",
"g-473",
"g-477",
"g-478",
"g-479",
"g-484",
"g-485",
"g-486",
"g-488",
"g-489",
"g-491",
"g-494",
"g-496",
"g-498",
"g-500",
"g-503",
"g-504",
"g-506",
"g-508",
"g-509",
"g-512",
"g-522",
"g-529",
"g-531",
"g-534",
"g-539",
"g-541",
"g-546",
"g-551",
"g-553",
"g-554",
"g-559",
"g-561",
"g-562",
"g-565",
"g-568",
"g-569",
"g-574",
"g-577",
"g-578",
"g-586",
"g-588",
"g-590",
"g-594",
"g-595",
"g-596",
"g-597",
"g-599",
"g-600",
"g-603",
"g-607",
"g-615",
"g-618",
"g-619",
"g-620",
"g-625",
"g-628",
"g-629",
"g-632",
"g-634",
"g-635",
"g-636",
"g-638",
"g-639",
"g-641",
"g-643",
"g-644",
"g-645",
"g-646",
"g-647",
"g-648",
"g-663",
"g-664",
"g-665",
"g-668",
"g-669",
"g-670",
"g-671",
"g-672",
"g-673",
"g-674",
"g-677",
"g-678",
"g-680",
"g-683",
"g-689",
"g-691",
"g-693",
"g-695",
"g-701",
"g-702",
"g-703",
"g-704",
"g-705",
"g-706",
"g-708",
"g-711",
"g-712",
"g-720",
"g-721",
"g-723",
"g-724",
"g-726",
"g-728",
"g-731",
"g-733",
"g-738",
"g-739",
"g-742",
"g-743",
"g-744",
"g-745",
"g-749",
"g-750",
"g-752",
"g-760",
"g-761",
"g-764",
"g-766",
"g-768",
"g-770",
"g-771",
"c-0",
"c-1",
"c-2",
"c-3",
"c-4",
"c-5",
"c-6",
"c-7",
"c-8",
"c-9",
"c-10",
"c-11",
"c-12",
"c-13",
"c-14",
"c-15",
"c-16",
"c-17",
"c-18",
"c-19",
"c-20",
"c-21",
"c-22",
"c-23",
"c-24",
"c-25",
"c-26",
"c-27",
"c-28",
"c-29",
"c-30",
"c-31",
"c-32",
"c-33",
"c-34",
"c-35",
"c-36",
"c-37",
"c-38",
"c-39",
"c-40",
"c-41",
"c-42",
"c-43",
"c-44",
"c-45",
"c-46",
"c-47",
"c-48",
"c-49",
"c-50",
"c-51",
"c-52",
"c-53",
"c-54",
"c-55",
"c-56",
"c-57",
"c-58",
"c-59",
"c-60",
"c-61",
"c-62",
"c-63",
"c-64",
"c-65",
"c-66",
"c-67",
"c-68",
"c-69",
"c-70",
"c-71",
"c-72",
"c-73",
"c-74",
"c-75",
"c-76",
"c-77",
"c-78",
"c-79",
"c-80",
"c-81",
"c-82",
"c-83",
"c-84",
"c-85",
"c-86",
"c-87",
"c-88",
"c-89",
"c-90",
"c-91",
"c-92",
"c-93",
"c-94",
"c-95",
"c-96",
"c-97",
"c-98",
"c-99",
]<set_options>
xgbFinal.fit(X_Full,y_Full )
batch_size = 32
n_seeds = 5
n_splits = 5
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )<prepare_x_and_y>
ypredXG = xgbFinal.predict(X_test_Frame )
%%time
denominator = counts.sum(axis=0)
 counts /= denominator
_, n_models = weights.shape
Y_preds = np.zeros(( n_models, test_size, n_classes))
for i in tqdm.trange(n_seeds):
for j in tqdm.trange(n_splits):
model = tf.keras.models.load_model(
 f".. /input/mlp-for-ensemble/kaggle_upload/rs/model_seed_{i}_fold_{j}.h5",
compile=False, 
)
 Y_preds[0] += counts[i * n_splits + j] * model.predict(
 [X_test_clipped, X_test_clipped[selected_columns]], batch_size=batch_size 
)
 tf.keras.backend.clear_session()
model = StackedTabNetClassifier(
 num_classes=n_classes,
num_features=n_clipped_features,
**params_stacked_tabnet, 
)
 model(np.zeros(( 1, n_clipped_features)))
 model.load_weights(
 f".. /input/mlp-for-ensemble/kaggle_upload/StackedTabNet/model_seed_{i}_fold_{j}.h5" 
)
 Y_preds[1] += counts[i * n_splits + j] * model.predict(
 X_test_clipped, batch_size=batch_size 
)
 tf.keras.backend.clear_session()
model = tf.keras.models.load_model(
 f".. /input/transformer-fit/Transformer_{i}_{j}.hdf5",
compile=False,
custom_objects={"TransformerEncoder": TransformerEncoderByZhang}, 
)
 Y_preds[2] += counts[i * n_splits + j] * model.predict(
 X_test_clipped, batch_size=batch_size 
)
 tf.keras.backend.clear_session()
model = Model(
 num_features=n_features_hirune924, num_targets=n_classes, hidden_size=1_500 
)
 model.load_state_dict(
 torch.load(
 f".. /input/pytorch-mlp-tabnet-many-fe-train/FOLD{j}_SEED{i}.pth",
map_location=torch.device(device), 
)
)
 model.to(device)
 Y_preds[3] += counts[i * n_splits + j] * predict_with_pytorch_model(
 model, X_test_hirune924, device, batch_size=test_size 
)
 with open(
 f".. /input/pytorch-mlp-tabnet-many-fe-train/tabnet_FOLD{j}_SEED{i}.pkl",
"rb", 
)as f:
model = pickle.load(f)
 Y_pred = model.predict(X_test_hirune924.values)
 Y_pred = 1.0 /(1.0 + np.exp(-Y_pred))
Y_preds[4] += counts[i * n_splits + j] * Y_pred
with open(
 f".. /input/pytorch-tabnet-pretraining-step3-many-fe-train/tabnet_FOLD{j}_SEED{i}.pkl",
"rb", 
)as f:
model = pickle.load(f)
 Y_pred = model.predict(X_test_hirune924.values)
 Y_pred = 1.0 /(1.0 + np.exp(-Y_pred))
Y_preds[5] += counts[i * n_splits + j] * Y_pred
model = tf.keras.models.load_model(
 f".. /input/resnetclassifier-fit/model_seed_{i}_fold_{j}.h5",
compile=False, 
)
 Y_preds[6] += counts[i * n_splits + j] * model.predict(
 X_test, batch_size=batch_size 
)
 tf.keras.backend.clear_session() 
<prepare_output>
Final_Pred = -4447 + np.exp(Ytest.inverse_transform(( ypredRidge*.20309379 + ypredLasso*.46724816 + ypredXG*.37162723)) )
Y_pred = pd.DataFrame(columns=Y.columns, index=X_test.index)
 for i, column in enumerate(columns):
Y_pred[column] = np.tensordot(weights[i], Y_preds[:, :, [i]], axes=(0, 0))
Y_pred[test_features["cp_type"] == "ctl_vehicle"] = 0.0<save_to_csv>
Final_Pred[np.exp(Ytest.inverse_transform(ypredRidge)) >y.max() ] = np.exp(Ytest.inverse_transform(ypredXG)) [np.exp(Ytest.inverse_transform(ypredRidge)) >y.max() ]
<import_modules><EOS> 
submission = pd.DataFrame({
"Id": IdTest,
"SalePrice": Final_Pred
})
 submission.to_csv('submission.csv', index=False )