Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<feature_engineering>
%matplotlib inline
dfk = pd.DataFrame({
'Kernel ID': ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'k'],
'Score': [ 0.69864 , 0.69846 , 0.69836 , 0.69824 , 0.69813, 0.69795, 0.69782, 0.69756, 0.69747, 0.69735, 0.69731],
'File Path': ['.. /input/tps-jan-2021-gbdts-baseline/submission.csv', '.. /input/pseudo-labelling/submission.csv', '.. /input/v4-baseline-lgb-no-tune/sub_0.6971.csv', '.. /input/tps21-optuna-lgb-fast-hyper-parameter-tunning/submission.csv', '.. /input/gbdts-baseline-prevision-io-for-free/submission.csv', '.. /input/v41-eda-gbdts/res41.csv', '.. /input/v3-ensemble-lgb-xgb-cat/submission.csv' , '.. /input/tabular-playground/sub_gbm.csv', '.. /input/v48tabular-playground-series-xgboost-lightgbm/V48-0.69747.csv', '.. /input/xgboost-hyperparameter-tuning-using-optuna/submission.csv', '.. /input/tabular-playground-some-slightly-useful-features/sub_gbm.csv']
})
 dfk
def keyword_disaster_probabilities(x):
tweets_w_keyword = np.sum(train["keyword"].fillna("" ).str.contains(x))
tweets_w_keyword_disaster = np.sum(train["keyword"].fillna("" ).str.contains(x)& train["target"] == 1)
 return tweets_w_keyword_disaster / tweets_w_keyword
keywords_vc["Disaster_Probability"] = keywords_vc.index.map(keyword_disaster_probabilities)
 keywords_vc.head()<sort_values>
def generate(main, support, coeff):
g = main.copy()
for i in main.columns[1:]:
res = []
target_main, target_support = [], []
target_main = main[i].tolist()
target_support = support[i].tolist()
for j in range(len(main)) :
res.append(( target_main[j] * coeff)+(target_support[j] *(1.- coeff)))
 g[i] = res
return g
keywords_vc.sort_values(by="Disaster_Probability", ascending=False ).head(10 )<count_values>
support = pd.read_csv(dfk.iloc[0, 2])
 for k in range(1, 7):
main = pd.read_csv(dfk.iloc[k, 2])
 support = generate(main, support, 0.7)
 sub1 = support
len(train["location"].value_counts() )<remove_duplicates>
main = pd.read_csv(dfk.iloc[9, 2])
 sub4 = generate(main, sub3, 0.55)
 drawing1(main, sub3, sub4 )
def create_corpus(target):
corpus = []
for w in train.loc[train["target"] == target]["text"].str.split() :
for i in w:
corpus.append(i)
 return corpus
def create_corpus_dict(target):
corpus = create_corpus(target)
 stop_dict = defaultdict(int)
 for word in corpus:
if word in stopwords:
stop_dict[word] += 1
return sorted(stop_dict.items() , key=lambda x:x[1], reverse=True )<count_duplicates>
sub = sub5
sub.to_csv("submission.csv", index=False)
 sub1.to_csv("submission1.csv", index=False)
 sub2.to_csv("submission2.csv", index=False)
 sub3.to_csv("submission3.csv", index=False)
 sub4.to_csv("submission3.csv", index=False)
 !ls