Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<choose_model_class>
%matplotlib inline
pd.set_option('max_colwidth', 40 )
MLA = [
ensemble.AdaBoostClassifier() ,
ensemble.BaggingClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
ensemble.RandomForestClassifier() ,
gaussian_process.GaussianProcessClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.PassiveAggressiveClassifier() ,
linear_model.RidgeClassifierCV() ,
linear_model.SGDClassifier() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
svm.SVC(probability=True),
svm.NuSVC(probability=True),
svm.LinearSVC() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
discriminant_analysis.LinearDiscriminantAnalysis() ,
discriminant_analysis.QuadraticDiscriminantAnalysis() ,
XGBClassifier()
]
cv_split = model_selection.ShuffleSplit(n_splits = 10, test_size =.3, train_size =.6, random_state = 0)
 MLA_columns = ['MLA Name', 'MLA Parameters','MLA Train Accuracy Mean', 'MLA Test Accuracy Mean', 'MLA Test Accuracy 3*STD' ,'MLA Time']
MLA_compare = pd.DataFrame(columns = MLA_columns)
 MLA_predict = data1[Target]
row_index = 0
for alg in MLA:
MLA_name = alg.__class__.__name__
MLA_compare.loc[row_index, 'MLA Name'] = MLA_name
MLA_compare.loc[row_index, 'MLA Parameters'] = str(alg.get_params())
 cv_results = model_selection.cross_validate(alg, data1[data1_x_bin], data1[Target], cv = cv_split)
 MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy Mean'] = cv_results['test_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy 3*STD'] = cv_results['test_score'].std() *3
alg.fit(data1[data1_x_bin], data1[Target])
 MLA_predict[MLA_name] = alg.predict(data1[data1_x_bin])
 row_index+=1
MLA_compare.sort_values(by = ['MLA Test Accuracy Mean'], ascending = False, inplace = True)
 MLA_compare 
<groupby>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()
pivot_female = data1[data1.Sex=='female'].groupby(['Sex','Pclass', 'Embarked','FareBin'])['Survived'].mean()
print('Survival Decision Tree w/Female Node:
',pivot_female)
 pivot_male = data1[data1.Sex=='male'].groupby(['Sex','Title'])['Survived'].mean()
print('
Survival Decision Tree w/Male Node:
',pivot_male )<create_dataframe>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1
def mytree(df):
Model = pd.DataFrame(data = {'Predict':[]})
 male_title = ['Master']
for index, row in df.iterrows() :
Model.loc[index, 'Predict'] = 0
if(df.loc[index, 'Sex'] == 'female'):
Model.loc[index, 'Predict'] = 1
if(( df.loc[index, 'Sex'] == 'female')& 
(df.loc[index, 'Pclass'] == 3)& 
(df.loc[index, 'Embarked'] == 'S')& 
(df.loc[index, 'Fare'] > 8)
):
Model.loc[index, 'Predict'] = 0
if(( df.loc[index, 'Sex'] == 'male')& 
(df.loc[index, 'Title'] in male_title)
):
Model.loc[index, 'Predict'] = 1
return Model
Tree_Predict = mytree(data1)
 print('Decision Tree Model Accuracy/Precision Score: {:.2f}%
'.format(metrics.accuracy_score(data1['Survived'], Tree_Predict)*100))
print(metrics.classification_report(data1['Survived'], Tree_Predict))<train_on_grid>
Dropout_new = 0.15
n_split = 5
lr = 3e-5
dtree = tree.DecisionTreeClassifier(random_state = 0)
 base_results = model_selection.cross_validate(dtree, data1[data1_x_bin], data1[Target], cv = cv_split)
 dtree.fit(data1[data1_x_bin], data1[Target])
 print('BEFORE DT Parameters: ', dtree.get_params())
 print("BEFORE DT Test w/bin score mean: {:.2f}".format(base_results['test_score'].mean() *100))
print("BEFORE DT Test w/bin score 3*std: +/- {:.2f}".format(base_results['test_score'].std() *100*3))
print('-'*10)
 param_grid = {'criterion': ['gini', 'entropy'],
'max_depth': [2,4,6,8,10,None],
'random_state': [0]
}
tune_model = model_selection.GridSearchCV(tree.DecisionTreeClassifier() , param_grid=param_grid, scoring = 'roc_auc', cv = cv_split)
 tune_model.fit(data1[data1_x_bin], data1[Target])
 print('AFTER DT Parameters: ', tune_model.best_params_)
 print("AFTER DT Test w/bin score mean: {:.2f}".format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER DT Test w/bin score 3*std: +/- {:.2f}".format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10 )<train_on_grid>
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=lr)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model
print('BEFORE DT RFE Training Shape Old: ', data1[data1_x_bin].shape)
 print('BEFORE DT RFE Training Columns Old: ', data1[data1_x_bin].columns.values)
 print("BEFORE DT RFE Test w/bin score mean: {:.2f}".format(base_results['test_score'].mean() *100))
print("BEFORE DT RFE Test w/bin score 3*std: +/- {:.2f}".format(base_results['test_score'].std() *100*3))
print('-'*10)
 dtree_rfe = feature_selection.RFECV(dtree, step = 1, scoring = 'accuracy', cv = cv_split)
 dtree_rfe.fit(data1[data1_x_bin], data1[Target])
 X_rfe = data1[data1_x_bin].columns.values[dtree_rfe.get_support() ]
rfe_results = model_selection.cross_validate(dtree, data1[X_rfe], data1[Target], cv = cv_split)
 print('AFTER DT RFE Training Shape New: ', data1[X_rfe].shape)
 print('AFTER DT RFE Training Columns New: ', X_rfe)
 print("AFTER DT RFE Test w/bin score mean: {:.2f}".format(rfe_results['test_score'].mean() *100))
print("AFTER DT RFE Test w/bin score 3*std: +/- {:.2f}".format(rfe_results['test_score'].std() *100*3))
print('-'*10)
 rfe_tune_model = model_selection.GridSearchCV(tree.DecisionTreeClassifier() , param_grid=param_grid, scoring = 'roc_auc', cv = cv_split)
 rfe_tune_model.fit(data1[X_rfe], data1[Target])
 print('AFTER DT RFE Tuned Parameters: ', rfe_tune_model.best_params_)
 print("AFTER DT RFE Tuned Test w/bin score mean: {:.2f}".format(rfe_tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER DT RFE Tuned Test w/bin score 3*std: +/- {:.2f}".format(rfe_tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10 )<choose_model_class>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
vote_est = [ 
('ada', ensemble.AdaBoostClassifier()), 
('bc', ensemble.BaggingClassifier()), 
('etc',ensemble.ExtraTreesClassifier()), 
('gbc', ensemble.GradientBoostingClassifier()), 
('rfc', ensemble.RandomForestClassifier()), 
('gpc', gaussian_process.GaussianProcessClassifier()), 
('lr', linear_model.LogisticRegressionCV()), 
('bnb', naive_bayes.BernoulliNB()), 
('gnb', naive_bayes.GaussianNB()), 
('knn', neighbors.KNeighborsClassifier()), 
('svc', svm.SVC(probability=True)) , 
('xgb', XGBClassifier())
 ]
vote_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 vote_hard_cv = model_selection.cross_validate(vote_hard, data1[data1_x_bin], data1[Target], cv = cv_split)
 vote_hard.fit(data1[data1_x_bin], data1[Target])
 print("Hard Voting Test w/bin score mean: {:.2f}".format(vote_hard_cv['test_score'].mean() *100))
print("Hard Voting Test w/bin score 3*std: +/- {:.2f}".format(vote_hard_cv['test_score'].std() *100*3))
print('-'*10)
 vote_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 vote_soft_cv = model_selection.cross_validate(vote_soft, data1[data1_x_bin], data1[Target], cv = cv_split)
 vote_soft.fit(data1[data1_x_bin], data1[Target])
 print("Soft Voting Test w/bin score mean: {:.2f}".format(vote_soft_cv['test_score'].mean() *100))
print("Soft Voting Test w/bin score 3*std: +/- {:.2f}".format(vote_soft_cv['test_score'].std() *100*3))
print('-'*10 )<train_on_grid>
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1
grid_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, data1[data1_x_bin], data1[Target], cv = cv_split)
 grid_hard.fit(data1[data1_x_bin], data1[Target])
 print("Hard Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_hard_cv['test_score'].mean() *100))
print("Hard Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_hard_cv['test_score'].std() *100*3))
print('-'*10)
 grid_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, data1[data1_x_bin], data1[Target], cv = cv_split)
 grid_soft.fit(data1[data1_x_bin], data1[Target])
 print("Soft Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_soft_cv['test_score'].mean() *100))
print("Soft Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_soft_cv['test_score'].std() *100*3))
print('-'*10 )<train_on_grid>
%%time
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_end_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=n_split,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting all Train for Outlier analysis...')
 preds_train = padded_model.predict([input_ids,attention_mask,token_type_ids],verbose=DISPLAY)
 preds_start_train += preds_train[0]/skf.n_splits
preds_end_train += preds_train[1]/skf.n_splits
print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
print(data_val.info())
 print("-"*10)
 data_val['Survived'] = mytree(data_val ).astype(int)
 data_val['Survived'] = grid_hard.predict(data_val[data1_x_bin])
 submit = data_val[['PassengerId','Survived']]
submit.to_csv(".. /working/submit.csv", index=False)
 print('Validation Data Distribution:
', data_val['Survived'].value_counts(normalize = True))
submit.sample(10 )<load_from_csv>
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
print(jac )
PATH_WEEK4 = '/kaggle/input/covid19-global-forecasting-week-4'
df_Train = pd.read_csv(f'{PATH_WEEK4}/train.csv', parse_dates=["Date"], engine='python')
 df_Test = pd.read_csv(f'{PATH_WEEK4}/test.csv', parse_dates=["Date"], engine='python')
<categorify>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )
def fillState(state, country):
if state == EMPTY_VAL: return country
return state<data_type_conversions>
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 test.sample(10 )
df_Train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_Test.rename(columns={'Country_Region':'Country'}, inplace=True)
 EMPTY_VAL = "EMPTY_VAL"
df_Train.rename(columns={'Province_State':'State'}, inplace=True)
 df_Train['State'].fillna(EMPTY_VAL, inplace=True)
 df_Train['State'] = df_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 df_Test.rename(columns={'Province_State':'State'}, inplace=True)
 df_Test['State'].fillna(EMPTY_VAL, inplace=True)
 df_Test['State'] = df_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1 )<prepare_x_and_y>
all = []
start = []
end = []
start_pred = []
end_pred = []
for k in range(input_ids.shape[0]):
a = np.argmax(preds_start_train[k,])
 b = np.argmax(preds_end_train[k,])
 start.append(np.argmax(start_tokens[k]))
end.append(np.argmax(end_tokens[k]))
if a>b:
st = train.loc[k,'text']
start_pred.append(0)
 end_pred.append(len(st))
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 start_pred.append(a)
 end_pred.append(b)
 all.append(st)
 train['start'] = start
train['end'] = end
train['start_pred'] = start_pred
train['end_pred'] = end_pred
train['selected_text_pred'] = all
train.sample(10 )
MIN_TEST_DATE = df_Test.Date.min()
df_train = df_Train.loc[df_Train.Date < MIN_TEST_DATE, :]
y1_Train = df_train.iloc[:, -2]
y2_Train = df_train.iloc[:, -1]<feature_engineering>
def metric_tse(df,col1,col2):
return df.apply(lambda x: jaccard(x[col1],x[col2]),axis=1 )
def extractDate(df, colName = 'Date'):

 assert colName in df.columns
df = df.assign(
 week = df.loc[:, colName].dt.week,
month = df.loc[:, colName].dt.month,
day = df.loc[:, colName].dt.day,
dayofweek = df.loc[:, colName].dt.dayofweek,
dayofyear = df.loc[:, colName].dt.dayofyear, 
)
 return df<data_type_conversions>
train = train.replace({'sentiment': {'negative': -1, 'neutral': 0, 'positive': 1}})
 train['len_text'] = train['text'].str.len()
train['len_selected_text'] = train['selected_text'].str.len()
train['diff_num'] = train['end']-train['start']
train['share'] = train['len_selected_text']/train['len_text']
def createNewDataset(df):

 df_New = df.copy()
df_New.loc[:, 'Date_Int'] =(df_New.loc[:, 'Date'].dt.strftime("%m%d")).astype('int16')
 df_New.drop(columns=['Date'], axis=1, inplace=True)
 return df_New<create_dataframe>
train['selected_text_pred'] = train['selected_text_pred'].map(lambda x: x.lstrip(' '))
train['len_selected_text_pred'] = train['selected_text_pred'].str.len()
train['diff_num_pred'] = train['end_pred']-train['start_pred']
train['share_pred'] = train['len_selected_text_pred']/train['len_text']
train['len_equal'] = 0
train.loc[(train['start'] == train['start_pred'])&(train['end'] == train['end_pred']), 'len_equal'] = 1
train['metric'] = metric_tse(train,'selected_text','selected_text_pred')
 train['res'] = 0
train.loc[train['metric'] == 1, 'res'] = 1
X_Train = createNewDataset(df_train)
 X_Test = createNewDataset(df_Test )<categorify>
def rep_3chr(text):
chr3 = 0
for word in text.split() :
for c in set(word):
if word.rfind(c+c+c)> -1:
chr3 = 1
return chr3
days_change = [1, 2, 3, 5, 7, 10]
def getChangeGrowth(df):
newCCols = []
newGCols = []
for day in days_change:
newCCol = f'C_{day}'
df['C'+newCCol] = df['LConfirmedCases'] - df[f'CD_{day}']
df['F'+newCCol] = df['LFatalities'] - df[f'FD_{day}']
newCCols.append(newCCol)
 newGCol = f'G_{day}'
df['C'+newGCol] = df['C'+newCCol] / df[f'CD_{day}']
df['F'+newGCol] = df['F'+newCCol] / df[f'FD_{day}']
newGCols.append(newGCol)
 df.fillna(0, inplace=True)
 return df<feature_engineering>
train['text_chr3'] = train['text'].apply(rep_3chr)
 train['selected_text_chr3'] = train['selected_text'].apply(rep_3chr)
 train['selected_text_pred_chr3'] = train['selected_text_pred'].apply(rep_3chr )
windows = [1, 2, 3, 5, 7]
def getMA(df):
newCMACols = []
newGMACols = []
for window in windows:
for day in days_change:
newCMACol = f'CMA_{day}_{window}'
df['C'+newCMACol] = df[f'CC_{day}'].rolling(window ).mean()
df['F'+newCMACol] = df[f'FC_{day}'].rolling(window ).mean()
newCMACols.append(newCMACol)
 newGMACol = f'GMA_{day}_{window}'
df['C'+newGMACol] = df[f'CG_{day}'].rolling(window ).mean()
df['F'+newGMACol] = df[f'FG_{day}'].rolling(window ).mean()
newGMACols.append(newGMACol)
 df.fillna(0, inplace=True)
 return df<groupby>
train_outlier = train[train['res'] == 0].reset_index(drop=True)
 train_outlier
cases = [1, 50, 100, 500, 1000, 5000, 35000, 75000, 100000]
def getCDSC(df):
newCDSCCols = []
for case in cases:
newDSCCol = f'{case}_CDSC'
df.loc[df.CD_1 == 0, newDSCCol] = 0
df.loc[df.CD_1 >= case, newDSCCol] = df[df.CD_1 >= case].groupby(['Country', 'State'] ).cumcount()
newCDSCCols.append(newDSCCol)
 df.fillna(0, inplace=True)
 return df<groupby>
sh_out = str(round(len(train_outlier)*100/len(train),1))
print('Number of outliers is ' + sh_out + '% from training data' )
deaths = [1, 50, 100, 500, 1000, 5000, 35000]
def getFDSC(df):
newFDSCCols = []
for death in deaths:
newDSCCol = f'{death}_FDSC'
df.loc[df.FD_1 == 0, newDSCCol] = 0
df.loc[df.FD_1 >= death, newDSCCol] = df[df.FD_1 >= death].groupby(['Country', 'State'] ).cumcount()
newFDSCCols.append(newDSCCol)
 df.fillna(0, inplace=True)
 return df<concatenate>
train_good = train[train['res'] == 1].reset_index(drop=True)
 train_good
df = pd.concat([df_Train, df_Test[df_Test.Date > df_Train.Date.max() ]], axis=0, sort=False, ignore_index=True )<feature_engineering>
train_outlier[train_outlier['diff_num']==0]
df['LConfirmedCases'] = np.log1p(df['ConfirmedCases'])
 df['LFatalities'] = np.log1p(df['Fatalities'] )<feature_engineering>
train_good[train_good['diff_num']==0]
df.loc[(df.Date >= df_Test.Date.min())&(df.Date <= df_Train.Date.max()), 'ForecastId'] = df_Test.loc[(df_Test.Date >= df_Test.Date.min())&(df_Test.Date <= df_Train.Date.max()), 'ForecastId'].values<statistical_test>
train_outlier[train_outlier['share']==1]
df = getDaysShift(df)
 df = getChangeGrowth(df)
 df = getMA(df)
 df = getCDSC(df)
 df = getFDSC(df )<groupby>
train_good[train_good['share']==1]
df['CSId'] = df.groupby(['Country', 'State'] ).cumcount()<count_unique_values>
len(train_outlier[train_outlier["text"].str.find(' ')== -1] )
df.loc[df.ForecastId > 0, 'ForecastId'].nunique()<filter>
len(train_good[train_good["text"].str.find(' ')== -1] )
df[df.Country =='Italy'][['ConfirmedCases', 'CD_1', 'CC_1']][70:95]<set_options>
def subtext_analysis(col, subtext, df1, str1, df2, str2):
result = pd.DataFrame(columns = ['subtext', str1, str2, 'share,%'])
 if(len(df1)> 0)and(len(df2)> 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num1 = len(df1[df1[col].str.find(subtext[i])> -1])
 result.loc[i, str1] = num1
num2 = len(df2[df2[col].str.find(subtext[i])> -1])
 result.loc[i, str2] = num2
result.loc[i,'share,%'] = round(num1*100/num2,1)if num2 != 0 else 0
print('Number of all data is', len(df2))
display(result.sort_values(by=['share,%', str1], ascending=False))
filterwarnings('ignore' )<categorify>
def subtext_analysis_one_df(col, subtext, df, str):
result = pd.DataFrame(columns = ['subtext', str, 'share of all,%'])
 num_all = len(df)
 if(num_all > 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num = len(df[df[col].str.find(subtext[i])> -1])
 result.loc[i, str] = num
result.loc[i,'share of all,%'] = round(num*100/num_all,1)
 print('Number of all data is', len(df))
display(result.sort_values(by='share of all,%', ascending=False))
cLEncoder = LabelEncoder()
sLEncoder = LabelEncoder()
df.loc[:, 'Country'] = cLEncoder.fit_transform(df.loc[:, 'Country'])
 df.loc[:, 'State'] = sLEncoder.fit_transform(df.loc[:, 'State'] )<data_type_conversions>
subtext_test = ['SAD', 'bullying', 'Uh', 'oh', 'onna', 'fun', 'addicted', 'Power', 'well', 'unhappy', 'funny', 'Tears', 'Fears', 'sleeeeepy', ' ', ',', '?', '!' ,'!!', '!!!', ':/', '...', 'http', '****']
X_Train = df[df.Date <= df_Train.Date.max() ]
X_Train.loc[:, 'Date_Int'] =(X_Train.loc[:, 'Date'].dt.strftime("%m%d")).astype('int16')
 yC_Train = X_Train.CC_1
yF_Train = X_Train.FC_1
X_Train = X_Train.drop(columns=['Id', 'ForecastId', 'Date', 'ConfirmedCases', 'Fatalities', 'CC_1', 'FC_1'])
 print(X_Train.shape, yC_Train.shape, yF_Train.shape )<choose_model_class>
subtext_analysis("selected_text", subtext_test, train_outlier, 'train_outliers', train, 'train_all' )
cLGBMR = LGBMRegressor(num_leaves=85,learning_rate=10**-1.89,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.1),min_child_samples=2,subsample=0.97,subsample_freq=10,
colsample_bytree=0.68,reg_lambda=10**1.4,random_state=7,n_jobs=-1)
 fLGBMR = LGBMRegressor(num_leaves=26,learning_rate=10**-1.63,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.04),min_child_samples=14,subsample=0.66,subsample_freq=5,
colsample_bytree=0.8,reg_lambda=10**1.92,random_state=7,n_jobs=-1 )<train_model>
subtext_analysis("selected_text", subtext_test, train_good, 'train_good', train, 'train_all' )
cLGBMR.fit(X_Train, yC_Train, categorical_feature=['Country','State'])
 fLGBMR.fit(X_Train, yF_Train, categorical_feature=['Country','State'] )<data_type_conversions>
subtext_analysis_one_df("selected_text", subtext_test, train, 'test_all' )
X_Test = df[df.Date >= df_Test.Date.min() ]
X_Test.loc[:, 'Date_Int'] =(X_Test.loc[:, 'Date'].dt.strftime("%m%d")).astype('int16')
 X_Test.drop(columns=['Id', 'ForecastId', 'Date', 'ConfirmedCases', 'Fatalities', 'CC_1', 'FC_1'], axis=1, inplace=True )<predict_on_test>
subtext_analysis_one_df("selected_text", subtext_test, test, 'test_all' )
nIds = range(X_Train.CSId.max() +1, X_Test.CSId.max() +1)
 yC_Pred = []
yF_Pred = []
for nId in nIds:
df = getDaysShift(df)
 df = getChangeGrowth(df)
 df = getMA(df)
 df = getCDSC(df)
 df = getFDSC(df)
 df.loc[df.CSId == nId, 'CC_1'] = cLGBMR.predict(X_Test[X_Test.CSId == nId])
 df.loc[df.CSId == nId,'LConfirmedCases'] = df.loc[df.CSId == nId, 'CC_1'] + df.loc[df.CSId == nId, 'CD_1']
df.loc[df.CSId == nId,'ConfirmedCases'] = np.exp(df.loc[df.CSId == nId,'LConfirmedCases'])- 1
df.loc[df.CSId == nId, 'FC_1'] = fLGBMR.predict(X_Test[X_Test.CSId == nId])
 df.loc[df.CSId == nId,'LFatalities'] = df.loc[df.CSId == nId, 'FC_1'] + df.loc[df.CSId == nId, 'FD_1']
df.loc[df.CSId == nId,'Fatalities'] = np.exp(df.loc[df.CSId == nId,'LFatalities'])- 1<categorify>
test['text_chr3'] = test['text'].apply(rep_3chr)
 test.head(10 )
df.loc[:, 'Country'] = cLEncoder.inverse_transform(df.loc[:, 'Country'])
 df.loc[:, 'State'] = sLEncoder.inverse_transform(df.loc[:, 'State'] )<filter>
train_outlier1 = train_outlier.nsmallest(1000, 'metric', keep='all')
 train_outlier2 = train_outlier.nsmallest(2000, 'metric', keep='all')
 train_outlier3 = train_outlier.nsmallest(3000, 'metric', keep='all')
 train_outlier5 = train_outlier.nsmallest(5000, 'metric', keep='all')
 train_outlier8 = train_outlier.nsmallest(8000, 'metric', keep='all' )
df[df.Country =='Italy'][['ConfirmedCases', 'CC_1', 'Fatalities', 'FC_1', 'CSId']][75:99]<filter>
subtext_analysis("selected_text", subtext_test, train_outlier1, 'in worst 1000 outliers', train_outlier, 'in all outliers' )
df_out = df.loc[df.ForecastId > 0, ['ForecastId', 'ConfirmedCases', 'Fatalities']]<data_type_conversions>
subtext_analysis("selected_text", subtext_test, train_outlier2, 'in worst 2000 outliers', train_outlier, 'in all outliers' )
df_out.ForecastId = df_out.ForecastId.astype('int' )<define_search_space>
subtext_analysis("selected_text", subtext_test, train_outlier3, 'in worst 3000 outliers', train_outlier, 'in all outliers' )
df_out.iloc[np.r_[1, 42, 45, 97, 143, 175, 267, 327, 350, 420, 450, 540, 590, 680, 730, 2880, 2900, 2960, 3000, 3050, 7000, 7500, 8000, 8500, 9000, 9500, 10000, 10500, 11000, 11500, 12000, 12500, 13000, -1], :]<save_to_csv>
subtext_analysis("selected_text", subtext_test, train_outlier5, 'in worst 5000 outliers', train_outlier, 'in all outliers' )
df_out.to_csv('submission.csv', index=False )<load_from_csv>
subtext_analysis("selected_text", subtext_test, train_outlier8, 'in worst 8000 outliers', train_outlier, 'in all outliers' )
PATH_WEEK4 = '/kaggle/input/covid19-global-forecasting-week-4'
df_Train = pd.read_csv(f'{PATH_WEEK4}/train.csv', parse_dates=["Date"], engine='python')
 df_Test = pd.read_csv(f'{PATH_WEEK4}/test.csv', parse_dates=["Date"], engine='python')
<categorify>
col_hist = ['sentiment', 'start', 'end', 'start_pred', 'end_pred', 'len_text', 'len_selected_text',
'text_chr3', 'selected_text_chr3', 'selected_text_pred_chr3', 'metric']
<data_type_conversions><EOS> 
data = train[['sentiment', 'start', 'end', 'start_pred', 'end_pred', 'len_text', 'len_selected_text', 'diff_num', 'share', 'metric', 'res']].dropna()
data