Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<train_model>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
print(train_df.info())
 print("<count_values>
train = pd.read_csv(".. /input/tabular-playground-series-jan-2021/train.csv")
 test = pd.read_csv(".. /input/tabular-playground-series-jan-2021/test.csv" )
train_df['target'].value_counts(normalize=True )<load_from_csv>
train[train['target']<4]
test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 test_df.head()<filter>
train_X, val_X, train_Y, val_Y = train_test_split(
 train_feature, train['target'], test_size=0.2, shuffle=True)
train_df[~train_df['keyword'].isnull() ][['keyword', 'text']]<count_unique_values>
cat = CatBoostRegressor(random_state = 7, loss_function='RMSE', verbose = False)
 cat.fit(train_X, train_Y)
 val_pred = cat.predict(val_X)
 score = np.sqrt(mean_squared_error(val_Y, val_pred))
print("CB model RMSE: ",end = "")
 print(score )
len(train_df['location'].unique() )<drop_column>
test_pred = cat.predict(test.drop("id",axis = 1))
submission = pd.DataFrame({
"id": test["id"],
"target":test_pred
})
 submission.to_csv('baseline_cat.csv', index=False )
train_df.drop(columns=['keyword', 'location'], inplace=True )<load_pretrained>
dtrain = lightgbm.Dataset(data=train_feature, label=train['target'])
 def hyp_lgbm(num_leaves, feature_fraction, bagging_fraction, max_depth, min_split_gain, min_child_weight, learning_rate):
params = {'application':'regression','num_iterations': 5000,
'early_stopping_round':100, 'metric':'rmse'}
params["num_leaves"] = int(round(num_leaves))
params['feature_fraction'] = max(min(feature_fraction, 1), 0)
 params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)
 params['max_depth'] = int(round(max_depth))
params['min_split_gain'] = min_split_gain
params['min_child_weight'] = min_child_weight
params['learning_rate'] = learning_rate
cv_result = lightgbm.cv(params, dtrain, nfold=3,
seed=7, stratified=False,
verbose_eval =None, metrics=['rmse'])
 return -np.min(cv_result['rmse-mean'])
spacy_en = spacy.load('en_core_web_sm', disable=['parser','ner'])
 bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True )<define_variables>
pds = {
'num_leaves':(5, 50),
'feature_fraction':(0.2, 1),
'bagging_fraction':(0.2, 1),
'max_depth':(2, 20),
'min_split_gain':(0.001, 0.1),
'min_child_weight':(10, 50),
'learning_rate':(0.01, 0.5),
}
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}<define_variables>
def cat_hyp(depth, bagging_temperature, l2_leaf_reg, learning_rate):
params = {"iterations": 100,
"loss_function": "RMSE",
"verbose": False}
params["depth"] = int(round(depth))
params["bagging_temperature"] = bagging_temperature
params["learning_rate"] = learning_rate
params["l2_leaf_reg"] = l2_leaf_reg
cat_feat = []
cv_dataset = cgb.Pool(data=train_feature, label=train['target'], cat_features=cat_feat)
 scores = cgb.cv(cv_dataset,
params,
fold_count=3)
 return -np.min(scores['test-RMSE-mean'] )
special_characters = {
"SuruÌ¤":"Suruc",
"JapÌ_n":"Japan" ,
"\x89ÛÏWhen":"When",
"å£3million":"3 million",
"fromåÊwounds":"from wounds",
"mÌ¼sica":"music",
"donå«t":"do not",
"didn`t":"did not",
"i\x89Ûªm":"I am",
"I\x89Ûªm":"I am",
"it\x89Ûªs":"it is",
"It\x89Ûªs":"It is",
"i\x89Ûªd":"I would",
"I\x89Ûªd":"I would",
"i\x89Ûªve":"I have",
"I\x89Ûªve":"I have",
"let\x89Ûªs":"let us",
"don\x89Ûªt":"do not",
"Don\x89Ûªt":"Do not",
"can\x89Ûªt":"cannot",
"Can\x89Ûªt":"Cannot",
"that\x89Ûªs":"that is",
"That\x89Ûªs":"That is",
"here\x89Ûªs":"here is",
"Here\x89Ûªs":"Here is",
"you\x89Ûªre":"you are",
"You\x89Ûªre":"You are",
"you\x89Ûªve":"you have",
"You\x89Ûªve":"You have",
"you\x89Ûªll":"you will",
"You\x89Ûªll":"You will",
"China\x89Ûªs":"China's",
"doesn\x89Ûªt":"does not",
"wouldn\x89Ûªt":"would not",
"\x89Û_":"",
"\x89Û¢":"",
"\x89ÛÒ":"",
"\x89ÛÓ":"",
"\x89ÛÏ":"",
"\x89Û÷":"",
"\x89Ûª":"",
"\x89Û¢åÊ":"",
"\x89Û\x9d":"",
"å_":"",
"å¨":"",
"åÀ":"",
"åÇ":"",
"åÊ":"",
"åÈ":"" ,
"Ì©":"",
"&lt;":"<",
"&gt;":">",
"&amp;":"&"
}<define_variables>
pds = {'depth':(3, 10),
'bagging_temperature':(0.1,10),
'l2_leaf_reg':(0.1, 10),
'learning_rate':(0.05, 0.3),
}
expand_contractions = {
"I'm":"I am",
"I'M":"I am",
"i'm":"I am",
"i'M":"I am",
"i'd":"I would",
"I'd":"I would",
"i'll":"I will",
"I'll":"I will",
"i've":"I have",
"I've":"I have",
"you're":"you are",
"You're":"You are",
"you'd":"you would",
"You'd":"You would",
"you've":"you have",
"You've":"You have",
"you'll":"you will",
"You'll":"You will" ,
"y'know":"you know" ,
"Y'know":"You know" ,
"y'all":"you all",
"Y'all":"You all",
"we're":"we are",
"We're":"We are",
"we've":"we have",
"We've":"We have" ,
"we'd":"we would",
"We'd":"We would",
"WE'VE":"We have",
"we'll":"we will",
"We'll":"We will",
"they're":"they are",
"They're":"They are",
"they'd":"they would",
"They'd":"They would" ,
"they've":"they have",
"They've":"They have",
"they'll":"they will",
"They'll":"They will",
"he's":"he is",
"He's":"He is",
"he'll":"he will",
"He'll":"He will",
"she's":"she is",
"She's":"She is",
"she'll":"she will",
"She'll":"She will",
"it's":"it is",
"It's":"It is",
"it'll":"it will",
"It'll":"It will",
"isn't":"is not",
"Isn't":"Is not",
"who's":"who is",
"Who's":"Who is",
"what's":"what is",
"What's":"What is",
"that's":"that is",
"That's":"That is",
"here's":"here is",
"Here's":"Here is",
"there's":"there is",
"There's":"There is",
"where's":"where is",
"Where's":"Where is" ,
"wHeRE's":"where is" ,
"how's":"how is" ,
"How's":"How is" ,
"how're":"how are" ,
"How're":"How are" ,
"let's":"let us",
"Let's":"Let us",
"won't":"will not",
"wasn't":"was not",
"aren't":"are not",
"couldn't":"could not",
"shouldn't":"should not",
"haven't":"have not",
"Haven't":"Have not",
"hasn't":"has not",
"wouldn't":"would not",
"weren't":"were not",
"Weren't":"Were not",
"ain't":"am not",
"Ain't":"am not",
"don't":"do not",
"Don't":"do not",
"DON'T":"Do not",
"didn't":"did not",
"Didn't":"Did not",
"DIDN'T":"Did not",
"doesn't":"does not",
"can't":"cannot",
"Can't":"Cannot",
"Could've":"Could have",
"should've":"should have",
"would've":"would have"
}<define_variables>
dtrain = xgb.DMatrix(train_feature, train['target'], feature_names=train_feature.columns.values)
 def hyp_xgb(max_depth, subsample, colsample_bytree,min_child_weight, gamma, learning_rate):
params = {
'objective': 'reg:squarederror',
'eval_metric':'rmse',
'nthread':-1
}
params['max_depth'] = int(round(max_depth))
params['subsample'] = max(min(subsample, 1), 0)
 params['colsample_bytree'] = max(min(colsample_bytree, 1), 0)
 params['min_child_weight'] = int(min_child_weight)
 params['gamma'] = max(gamma, 0)
 params['learning_rate'] = learning_rate
scores = xgb.cv(params, dtrain, num_boost_round=500,verbose_eval=False,
early_stopping_rounds=10, nfold=3)
 return -scores['test-rmse-mean'].iloc[-1]
informal_abbreviations = {
"b/c":"because",
"w/e":"whatever",
"w/out":"without",
"w/o":"without",
"w/":"with ",
"<3":"love",
"c/o":"care of",
"p/u":"pick up",
"
":" "
}<string_transform>
pds ={
'min_child_weight':(3, 20),
'gamma':(0, 5),
'subsample':(0.7, 1),
'colsample_bytree':(0.1, 1),
'max_depth':(3, 10),
'learning_rate':(0.01, 0.5)
 }
def clean_text(text):
cleaned_text = text.lower()
cleaned_text = re.sub(r'https?:\S+|www\.\S+', '', cleaned_text)
 cleaned_text = re.sub(r'<.*?>', '', cleaned_text)
 cleaned_text = ''.join(ch for ch in cleaned_text if ch in string.printable)
 cleaned_text = ' '.join(abbreviations[word] if word in abbreviations else word for word in cleaned_text.split())
 cleaned_text = re.sub(r'@\S+', 'USER', cleaned_text)
 cleaned_text = re.sub(r'\d+?[.,\d]*', 'NUMBER', cleaned_text)
 emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 cleaned_text = emoji_pattern.sub(r'', cleaned_text)
 cleaned_text = re.sub(r'\s+', ' ', cleaned_text ).strip()
cleaned_text = word_tokenize(cleaned_text)
 cleaned_text = [abbreviations[word] if word in abbreviations else word for word in cleaned_text]
cleaned_text = [special_characters[word] if word in special_characters else word for word in cleaned_text]
cleaned_text = [expand_contractions[word] if word in expand_contractions else word for word in cleaned_text]
cleaned_text = [informal_abbreviations[word] if word in informal_abbreviations else word for word in cleaned_text]
cleaned_text = [smileys[word] if word in smileys else word for word in cleaned_text]
cleaned_text = ' '.join([word for word in cleaned_text])
 doc = spacy_en(cleaned_text)
 cleaned_text = ' '.join([token.lemma_ for token in doc if token.lemma_ not in ['-PRON-'] and token.pos_ not in ['DET','-PRON']])
 return cleaned_text<string_transform>
param_lgbm = {
'bagging_fraction': 0.973905385549851,
'feature_fraction': 0.2945585590881137,
'learning_rate': 0.03750332268701348,
'max_depth': int(7.66),
'min_child_weight': int(41.36),
'min_split_gain': 0.04033836353603582,
'num_leaves': int(46.42),
'application':'regression',
'num_iterations': 5000,
'metric': 'rmse'
}
param_cat = {
'bagging_temperature': 0.31768713094131684,
'depth': int(8.03),
'l2_leaf_reg': 1.3525686450404295,
'learning_rate': 0.18,
'iterations': 150,
'loss_function': 'RMSE',
'verbose': False
}
param_xgb = {
'colsample_bytree': 0.8119098377889549,
'gamma': 2.244423418642122,
'learning_rate': 0.015800631696721114,
'max_depth': int(9.846),
'min_child_weight': int(15.664),
'subsample': 0.82345,
'objective': 'reg:squarederror',
'eval_metric':'rmse',
'num_boost_roun' : 500
}
def add_special_token(text):
cleaned_text = "[CLS] " + text + " [SEP]"
return cleaned_text<feature_engineering>
estimators = [ 
('lgbm', lightgbm.LGBMRegressor(**param_lgbm, random_state=7, n_jobs=-1)) , 
('xgbr', XGBRegressor(**param_xgb, random_state=7, nthread=-1)) , 
('cat', CatBoostRegressor(**param_cat))
]
reg = StackingRegressor(
 estimators=estimators,
final_estimator=lightgbm.LGBMRegressor() ,
n_jobs=-1,
cv=5 
)
 train_X, val_X, train_Y, val_Y = train_test_split(
 train_feature, train['target'], test_size=0.2, shuffle=True)
 reg.fit(train_X,train_Y)
 val_pred = reg.predict(val_X)
 score = np.sqrt(mean_squared_error(val_Y, val_pred))
print("Final model RMSE: ",end = "")
 print(score)
train_df['cleaned_text'] = np.vectorize(clean_text )(train_df['text'])
 train_df['cleaned_text'] = np.vectorize(add_special_token )(train_df['cleaned_text'] )<define_variables>
reg = StackingRegressor(
 estimators=estimators,
final_estimator=lightgbm.LGBMRegressor() ,
n_jobs=-1,
cv=5 
)
 reg.fit(train_feature, train['target'])
MAX_LEN = 128
BATCH_SZ=32<string_transform>
test_pred = reg.predict(test.drop("id",axis = 1))
submission = pd.DataFrame({
"id": test["id"],
"target":test_pred
})
 submission.to_csv('stacking_sub.csv', index=False )