Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<rename_columns>
%matplotlib inline
pd.set_option('max_colwidth', 40 )
train['Province_State'].fillna('No Data',inplace=True)
 test['Province_State'].fillna('No Data',inplace=True )<data_type_conversions>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()
train['Date']= pd.to_datetime(train['Date'])
 test['Date']= pd.to_datetime(test['Date'] )<feature_engineering>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1
def create_time_features(df):
df['date'] = df['Date']
df['hour'] = df['date'].dt.hour
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['month'] = df['date'].dt.month
df['year'] = df['date'].dt.year
df['dayofyear'] = df['date'].dt.dayofyear
df['dayofmonth'] = df['date'].dt.day
df['weekofyear'] = df['date'].dt.weekofyear
X = df[['hour','dayofweek','quarter','month','year',
'dayofyear','dayofmonth','weekofyear']]
return X
create_time_features(train)
 create_time_features(test)
 train=train.drop(columns=['Date'],axis=1)
 test=test.drop(columns=['Date'],axis=1)
 train=train.drop(columns=['date'],axis=1)
 test=test.drop(columns=['date'],axis=1 )<categorify>
Dropout_new = 0.15
n_split = 5
lr = 3e-5
train_dummies_Province_State = pd.get_dummies(train['Province_State'])
 test_dummies_Province_State = pd.get_dummies(test['Province_State'])
 train_dummies_Country_Region = pd.get_dummies(train['Country_Region'])
 test_dummies_Country_Region = pd.get_dummies(test['Country_Region'])
 train=train.drop(['Country_Region','Province_State','Id'],axis=1)
 test=test.drop(['Country_Region','Province_State','ForecastId'],axis=1)
 train=pd.concat([train,train_dummies_Province_State,train_dummies_Country_Region],axis=1)
 test=pd.concat([test,test_dummies_Province_State,test_dummies_Country_Region],axis=1)
 train.head()<choose_model_class>
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=lr)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model
model = DecisionTreeRegressor(criterion='mse', splitter='best' )<train_model>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
model.fit(features,target1 )<predict_on_test>
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1
submission['ConfirmedCases'] = model.predict(test )<train_model>
%%time
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_end_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=n_split,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting all Train for Outlier analysis...')
 preds_train = padded_model.predict([input_ids,attention_mask,token_type_ids],verbose=DISPLAY)
 preds_start_train += preds_train[0]/skf.n_splits
preds_end_train += preds_train[1]/skf.n_splits
print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
model.fit(features,target2)
 submission['Fatalities'] = model.predict(test )<data_type_conversions>
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
print(jac )
submission.round().astype(int )<save_to_csv>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )
submission.to_csv('submission.csv',index=False )<set_options>
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 test.sample(10 )
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99)
<set_options>
all = []
start = []
end = []
start_pred = []
end_pred = []
for k in range(input_ids.shape[0]):
a = np.argmax(preds_start_train[k,])
 b = np.argmax(preds_end_train[k,])
 start.append(np.argmax(start_tokens[k]))
end.append(np.argmax(end_tokens[k]))
if a>b:
st = train.loc[k,'text']
start_pred.append(0)
 end_pred.append(len(st))
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 start_pred.append(a)
 end_pred.append(b)
 all.append(st)
 train['start'] = start
train['end'] = end
train['start_pred'] = start_pred
train['end_pred'] = end_pred
train['selected_text_pred'] = all
train.sample(10 )
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
sns.set_palette(sns.color_palette('tab20', 20)) 
<load_from_csv>
def metric_tse(df,col1,col2):
return df.apply(lambda x: jaccard(x[col1],x[col2]),axis=1 )
COMP = '.. /input/covid19-global-forecasting-week-2'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '')
 train['Location'] = train['Country_Region'] + '-' + train['Province_State'].fillna('')
 test['Location'] = test['Country_Region'] + '-' + test['Province_State'].fillna('')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 country_codes = pd.read_csv('.. /input/covid19-metadata/country_codes.csv', keep_default_na=False)
 train = train.merge(country_codes, on='Country_Region', how='left')
 test = test.merge(country_codes, on='Country_Region', how='left')
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in tqdm(df.groupby('Location')) :
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1 
<save_to_csv>
train = train.replace({'sentiment': {'negative': -1, 'neutral': 0, 'positive': 1}})
 train['len_text'] = train['text'].str.len()
train['len_selected_text'] = train['selected_text'].str.len()
train['diff_num'] = train['end']-train['start']
train['share'] = train['len_selected_text']/train['len_text']
train = train.sort_values(by='Date')
 countries_latest_state = train[train['Date'] == TRAIN_END].groupby([
'Country_Region', 'continent', 'geo_region', 'country_iso_code_3'] ).sum() [[
'ConfirmedCases', 'Fatalities']].reset_index()
countries_latest_state['Log10Confirmed'] = np.log10(countries_latest_state.ConfirmedCases + 1)
 countries_latest_state['Log10Fatalities'] = np.log10(countries_latest_state.Fatalities + 1)
 countries_latest_state = countries_latest_state.sort_values(by='Fatalities', ascending=False)
 countries_latest_state.to_csv('countries_latest_state.csv', index=False)
 countries_latest_state.shape
countries_latest_state.head()<merge>
train['selected_text_pred'] = train['selected_text_pred'].map(lambda x: x.lstrip(' '))
train['len_selected_text_pred'] = train['selected_text_pred'].str.len()
train['diff_num_pred'] = train['end_pred']-train['start_pred']
train['share_pred'] = train['len_selected_text_pred']/train['len_text']
train['len_equal'] = 0
train.loc[(train['start'] == train['start_pred'])&(train['end'] == train['end_pred']), 'len_equal'] = 1
train['metric'] = metric_tse(train,'selected_text','selected_text_pred')
 train['res'] = 0
train.loc[train['metric'] == 1, 'res'] = 1
latest_loc = train[train['Date'] == TRAIN_END][['Location', 'ConfirmedCases', 'Fatalities']]
max_loc = train.groupby(['Location'])[['ConfirmedCases', 'Fatalities']].max().reset_index()
check = pd.merge(latest_loc, max_loc, on='Location')
 np.mean(check.ConfirmedCases_x == check.ConfirmedCases_y)
 np.mean(check.Fatalities_x == check.Fatalities_y)
 check[check.Fatalities_x != check.Fatalities_y]
check[check.ConfirmedCases_x != check.ConfirmedCases_y]<feature_engineering>
def rep_3chr(text):
chr3 = 0
for word in text.split() :
for c in set(word):
if word.rfind(c+c+c)> -1:
chr3 = 1
return chr3
regional_progress = train_clean.groupby(['DateTime', 'continent'] ).sum() [['ConfirmedCases', 'Fatalities']].reset_index()
regional_progress['Log10Confirmed'] = np.log10(regional_progress.ConfirmedCases + 1)
 regional_progress['Log10Fatalities'] = np.log10(regional_progress.Fatalities + 1)
 regional_progress = regional_progress[regional_progress.continent != '<merge>
train['text_chr3'] = train['text'].apply(rep_3chr)
 train['selected_text_chr3'] = train['selected_text'].apply(rep_3chr)
 train['selected_text_pred_chr3'] = train['selected_text_pred'].apply(rep_3chr )
countries_0301 = country_progress[country_progress.Date == '2020-03-01'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_0331 = country_progress[country_progress.Date == '2020-03-31'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_in_march = pd.merge(countries_0301, countries_0331, on='Country_Region', suffixes=['_0301', '_0331'])
 countries_in_march['IncreaseInMarch'] = countries_in_march.ConfirmedCases_0331 /(countries_in_march.ConfirmedCases_0301 + 1)
 countries_in_march = countries_in_march[countries_in_march.ConfirmedCases_0331 > 200].sort_values(
 by='IncreaseInMarch', ascending=False)
 countries_in_march.tail(15 )<save_to_csv>
train_outlier = train[train['res'] == 0].reset_index(drop=True)
 train_outlier
train_clean['Geo
latest = train_clean[train_clean.Date == '2020-03-31'][[
'Geo
daily_confirmed_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_confirmed_deltas = latest.merge(daily_confirmed_deltas, on='Geo
daily_confirmed_deltas.shape
daily_confirmed_deltas.head()
daily_confirmed_deltas.to_csv('daily_confirmed_deltas.csv', index=False )<save_to_csv>
sh_out = str(round(len(train_outlier)*100/len(train),1))
print('Number of outliers is ' + sh_out + '% from training data' )
deltas = train_clean[np.logical_and(
 train_clean.LogConfirmed > 2,
~train_clean.Location.str.startswith('China')
)].dropna().sort_values(by='LogConfirmedDelta', ascending=False)
 deltas['start'] = deltas['LogConfirmed'].round(0)
 confirmed_deltas = pd.concat([
deltas.groupby('start')[['LogConfirmedDelta']].mean() ,
deltas.groupby('start')[['LogConfirmedDelta']].std() ,
deltas.groupby('start')[['LogConfirmedDelta']].count()
], axis=1)
 deltas.mean()
confirmed_deltas.columns = ['avg', 'std', 'cnt']
confirmed_deltas
confirmed_deltas.to_csv('confirmed_deltas.csv' )<feature_engineering>
train_good = train[train['res'] == 1].reset_index(drop=True)
 train_good
DECAY = 0.93
DECAY ** 7, DECAY ** 14, DECAY ** 21, DECAY ** 28
confirmed_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
confirmed_deltas['DELTA'] = GLOBAL_DELTA
confirmed_deltas.loc[confirmed_deltas.continent=='Africa', 'DELTA'] = 0.14
confirmed_deltas.loc[confirmed_deltas.continent=='Oceania', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Korea South', 'DELTA'] = 0.011
confirmed_deltas.loc[confirmed_deltas.Country_Region=='US', 'DELTA'] = 0.15
confirmed_deltas.loc[confirmed_deltas.Country_Region=='China', 'DELTA'] = 0.01
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Japan', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Norway', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iceland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Austria', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Italy', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Spain', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Portugal', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Israel', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iran', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Germany', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Russia', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Brazil', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Turkey', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Philippines', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Location=='France-', 'DELTA'] = 0.1
confirmed_deltas.loc[confirmed_deltas.Location=='United Kingdom-', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
confirmed_deltas.loc[confirmed_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Location=='San Marino-', 'DELTA'] = 0.03
confirmed_deltas.shape, confirmed_deltas.DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].shape, confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA]
confirmed_deltas.describe()<save_to_csv>
train_outlier[train_outlier['diff_num']==0]
daily_log_confirmed = train_clean.pivot('Location', 'Date', 'LogConfirmed' ).reset_index()
daily_log_confirmed = daily_log_confirmed.sort_values(TRAIN_END, ascending=False)
 daily_log_confirmed.to_csv('daily_log_confirmed.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in confirmed_deltas.Location.values:
confirmed_delta = confirmed_deltas.loc[confirmed_deltas.Location == loc, 'DELTA'].values[0]
daily_log_confirmed.loc[daily_log_confirmed.Location == loc, new_day] = daily_log_confirmed.loc[daily_log_confirmed.Location == loc, last_day] + \
confirmed_delta * DECAY ** i<save_to_csv>
train_good[train_good['diff_num']==0]
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_death_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_death_deltas = latest.merge(daily_death_deltas, on='Geo
daily_death_deltas.shape
daily_death_deltas.head()
daily_death_deltas.to_csv('daily_death_deltas.csv', index=False )<feature_engineering>
train_outlier[train_outlier['share']==1]
death_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
death_deltas['DELTA'] = GLOBAL_DELTA
death_deltas.loc[death_deltas.Country_Region=='China', 'DELTA'] = 0.005
death_deltas.loc[death_deltas.continent=='Oceania', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Country_Region=='Korea South', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Japan', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='US', 'DELTA'] = 0.17
death_deltas.loc[death_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Norway', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Iceland', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Country_Region=='Austria', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Italy', 'DELTA'] = 0.07
death_deltas.loc[death_deltas.Country_Region=='Spain', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Country_Region=='Portugal', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Israel', 'DELTA'] = 0.16
death_deltas.loc[death_deltas.Country_Region=='Iran', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='Germany', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Russia', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Brazil', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Turkey', 'DELTA'] = 0.22
death_deltas.loc[death_deltas.Country_Region=='Philippines', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Location=='France-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='United Kingdom-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
death_deltas.loc[death_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Location=='San Marino-', 'DELTA'] = 0.05
death_deltas.shape
death_deltas.DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].shape
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA]
death_deltas.describe()<save_to_csv>
train_good[train_good['share']==1]
daily_log_deaths = train_clean.pivot('Location', 'Date', 'LogFatalities' ).reset_index()
daily_log_deaths = daily_log_deaths.sort_values(TRAIN_END, ascending=False)
 daily_log_deaths.to_csv('daily_log_deaths.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in death_deltas.Location:
death_delta = death_deltas.loc[death_deltas.Location == loc, 'DELTA'].values[0]
daily_log_deaths.loc[daily_log_deaths.Location == loc, new_day] = daily_log_deaths.loc[daily_log_deaths.Location == loc, last_day] + \
death_delta * DECAY ** i<feature_engineering>
len(train_outlier[train_outlier["text"].str.find(' ')== -1] )
confirmed = []
fatalities = []
for id, d, loc in tqdm(test[['ForecastId', 'Date', 'Location']].values):
c = to_exp(daily_log_confirmed.loc[daily_log_confirmed.Location == loc, d].values[0])
 f = to_exp(daily_log_deaths.loc[daily_log_deaths.Location == loc, d].values[0])
 confirmed.append(c)
 fatalities.append(f )<prepare_output>
len(train_good[train_good["text"].str.find(' ')== -1] )
my_submission = test.copy()
my_submission['ConfirmedCases'] = confirmed
my_submission['Fatalities'] = fatalities
my_submission.shape
my_submission.head() 
<save_to_csv>
def subtext_analysis(col, subtext, df1, str1, df2, str2):
result = pd.DataFrame(columns = ['subtext', str1, str2, 'share,%'])
 if(len(df1)> 0)and(len(df2)> 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num1 = len(df1[df1[col].str.find(subtext[i])> -1])
 result.loc[i, str1] = num1
num2 = len(df2[df2[col].str.find(subtext[i])> -1])
 result.loc[i, str2] = num2
result.loc[i,'share,%'] = round(num1*100/num2,1)if num2 != 0 else 0
print('Number of all data is', len(df2))
display(result.sort_values(by=['share,%', str1], ascending=False))
my_submission[[
'ForecastId', 'ConfirmedCases', 'Fatalities'
]].to_csv('submission.csv', index=False)
 print(DECAY)
 my_submission.head()
my_submission.tail()
my_submission.shape<train_model>
def subtext_analysis_one_df(col, subtext, df, str):
result = pd.DataFrame(columns = ['subtext', str, 'share of all,%'])
 num_all = len(df)
 if(num_all > 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num = len(df[df[col].str.find(subtext[i])> -1])
 result.loc[i, str] = num
result.loc[i,'share of all,%'] = round(num*100/num_all,1)
 print('Number of all data is', len(df))
display(result.sort_values(by='share of all,%', ascending=False))
end = dt.datetime.now()
print('Finished', end,(end - start ).seconds, 's' )<import_modules>
subtext_test = ['SAD', 'bullying', 'Uh', 'oh', 'onna', 'fun', 'addicted', 'Power', 'well', 'unhappy', 'funny', 'Tears', 'Fears', 'sleeeeepy', ' ', ',', '?', '!' ,'!!', '!!!', ':/', '...', 'http', '****']
import pandas as pd
import datetime
import lightgbm as lgb
import numpy as np
from sklearn import preprocessing<load_from_csv>
subtext_analysis("selected_text", subtext_test, train_outlier, 'train_outliers', train, 'train_all' )
train = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-3/test.csv")
 sub = pd.read_csv(".. /input/covid19-global-forecasting-week-3/submission.csv" )<concatenate>
subtext_analysis("selected_text", subtext_test, train_good, 'train_good', train, 'train_all' )
train = train.append(test[test['Date']>'2020-04-07'] )<data_type_conversions>
subtext_analysis_one_df("selected_text", subtext_test, train, 'test_all' )
train['Date'] = pd.to_datetime(train['Date'], format='%Y-%m-%d' )<feature_engineering>
subtext_analysis_one_df("selected_text", subtext_test, test, 'test_all' )
train['day_dist'] = train['Date']-train['Date'].min()<feature_engineering>
test['text_chr3'] = test['text'].apply(rep_3chr)
 test.head(10 )
train['day_dist'] = train['day_dist'].dt.days<define_variables>
train_outlier1 = train_outlier.nsmallest(1000, 'metric', keep='all')
 train_outlier2 = train_outlier.nsmallest(2000, 'metric', keep='all')
 train_outlier3 = train_outlier.nsmallest(3000, 'metric', keep='all')
 train_outlier5 = train_outlier.nsmallest(5000, 'metric', keep='all')
 train_outlier8 = train_outlier.nsmallest(8000, 'metric', keep='all' )
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols<correct_missing_values>
subtext_analysis("selected_text", subtext_test, train_outlier1, 'in worst 1000 outliers', train_outlier, 'in all outliers' )
for cat_col in cat_cols:
train[cat_col].fillna('no_value', inplace = True )<feature_engineering>
subtext_analysis("selected_text", subtext_test, train_outlier2, 'in worst 2000 outliers', train_outlier, 'in all outliers' )
train['place'] = train['Province_State']+'_'+train['Country_Region'] 
<define_variables>
subtext_analysis("selected_text", subtext_test, train_outlier3, 'in worst 3000 outliers', train_outlier, 'in all outliers' )
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols<categorify>
subtext_analysis("selected_text", subtext_test, train_outlier5, 'in worst 5000 outliers', train_outlier, 'in all outliers' )
for cat_col in ['place']:
le = preprocessing.LabelEncoder()
le.fit(train[cat_col])
 train[cat_col]=le.transform(train[cat_col] )<define_variables>
subtext_analysis("selected_text", subtext_test, train_outlier8, 'in worst 8000 outliers', train_outlier, 'in all outliers' )
drop_cols = ['Id','ForecastId', 'ConfirmedCases','Date', 'Fatalities',
'day_dist', 'Province_State', 'Country_Region']<filter>
col_hist = ['sentiment', 'start', 'end', 'start_pred', 'end_pred', 'len_text', 'len_selected_text',
'text_chr3', 'selected_text_chr3', 'selected_text_pred_chr3', 'metric']
val = train[(train['Date']>='2020-03-20')&(train['Id'].isnull() ==False)] 
<prepare_x_and_y>
data = train[['sentiment', 'start', 'end', 'start_pred', 'end_pred', 'len_text', 'len_selected_text', 'diff_num', 'share', 'metric', 'res']].dropna()
data
y_ft = train["Fatalities"]
y_val_ft = val["Fatalities"]
y_cc = train["ConfirmedCases"]
y_val_cc = val["ConfirmedCases"] 
<compute_test_metric>
%matplotlib inline
pd.set_option('max_colwidth', 40 )
def rmsle(y_true, y_pred):
return np.sqrt(np.mean(np.power(np.log1p(y_pred)- np.log1p(y_true), 2)) )<compute_test_metric>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()
def mape(y_true, y_pred):
return np.mean(np.abs(y_pred -y_true)*100/(y_true+1))<filter>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1
dates = dates[dates>'2020-04-07']<init_hyperparams>
Dropout_new = 0.15
n_split = 5
lr = 3e-5
params = {
"objective": "regression",
"boosting": 'gbdt',
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.9,
"reg_lambda": 2,
"metric": "rmse",
'min_data_in_leaf':20
}<filter>
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=lr)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model
train[train['Date']==date]<filter>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
test[test['Country_Region']=='Italy']<filter>
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1
test[(test['Country_Region']=='China')&(test['Province_State']=='Zhejiang')]<filter>
%%time
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_end_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=n_split,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting all Train for Outlier analysis...')
 preds_train = padded_model.predict([input_ids,attention_mask,token_type_ids],verbose=DISPLAY)
 preds_start_train += preds_train[0]/skf.n_splits
preds_end_train += preds_train[1]/skf.n_splits
print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
test[test['Country_Region']=='Italy']<load_from_csv>
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
print(jac )
train_sub = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv" )<merge>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )
test = pd.merge(test,train_sub[['Province_State','Country_Region',
'Date','ConfirmedCases',
'Fatalities']],
on=['Province_State','Country_Region',
'Date'], how='left' )<filter>
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 test.sample(10 )
test.loc[test['ConfirmedCases_x'].isnull() ==True]<feature_engineering>
all = []
start = []
end = []
start_pred = []
end_pred = []
for k in range(input_ids.shape[0]):
a = np.argmax(preds_start_train[k,])
 b = np.argmax(preds_end_train[k,])
 start.append(np.argmax(start_tokens[k]))
end.append(np.argmax(end_tokens[k]))
if a>b:
st = train.loc[k,'text']
start_pred.append(0)
 end_pred.append(len(st))
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 start_pred.append(a)
 end_pred.append(b)
 all.append(st)
 train['start'] = start
train['end'] = end
train['start_pred'] = start_pred
train['end_pred'] = end_pred
train['selected_text_pred'] = all
train.sample(10 )
test.loc[test['ConfirmedCases_x'].isnull() ==True, 'ConfirmedCases_x'] =test.loc[test['ConfirmedCases_x'].isnull() ==True, 'ConfirmedCases_y']<feature_engineering>
def metric_tse(df,col1,col2):
return df.apply(lambda x: jaccard(x[col1],x[col2]),axis=1 )
test.loc[test['Fatalities_x'].isnull() ==True, 'Fatalities_x'] = test.loc[test['Fatalities_x'].isnull() ==True, 'Fatalities_y']<filter>
train = train.replace({'sentiment': {'negative': -1, 'neutral': 0, 'positive': 1}})
 train['len_text'] = train['text'].str.len()
train['len_selected_text'] = train['selected_text'].str.len()
train['diff_num'] = train['end']-train['start']
train['share'] = train['len_selected_text']/train['len_text']
last_amount = test.loc[(test['Country_Region']=='Italy')&(test['Date']=='2020-04-07'),'ConfirmedCases_x']
last_fat = test.loc[(test['Country_Region']=='Italy')&(test['Date']=='2020-04-07'),'Fatalities_x']<define_variables>
train['selected_text_pred'] = train['selected_text_pred'].map(lambda x: x.lstrip(' '))
train['len_selected_text_pred'] = train['selected_text_pred'].str.len()
train['diff_num_pred'] = train['end_pred']-train['start_pred']
train['share_pred'] = train['len_selected_text_pred']/train['len_text']
train['len_equal'] = 0
train.loc[(train['start'] == train['start_pred'])&(train['end'] == train['end_pred']), 'len_equal'] = 1
train['metric'] = metric_tse(train,'selected_text','selected_text_pred')
 train['res'] = 0
train.loc[train['metric'] == 1, 'res'] = 1
i = 0
k = 30<filter>
def rep_3chr(text):
chr3 = 0
for word in text.split() :
for c in set(word):
if word.rfind(c+c+c)> -1:
chr3 = 1
return chr3
test.loc[(test['Country_Region']=='Italy')]<feature_engineering>
train['text_chr3'] = train['text'].apply(rep_3chr)
 train['selected_text_chr3'] = train['selected_text'].apply(rep_3chr)
 train['selected_text_pred_chr3'] = train['selected_text_pred'].apply(rep_3chr )
for date in dates:
k = k-1
i = i+1
test.loc[(test['Country_Region']=='Italy')&(test['Date']==date),
'ConfirmedCases_x']=last_amount.values[0] + i*(5000-(100*i))
test.loc[(test['Country_Region']=='Italy')&(test['Date']==date),
'Fatalities_x'] = last_fat.values[0]+i*(800-(10*i))<filter>
train_outlier = train[train['res'] == 0].reset_index(drop=True)
 train_outlier
test.loc[(test['Country_Region']=='Spain')]<feature_engineering>
sh_out = str(round(len(train_outlier)*100/len(train),1))
print('Number of outliers is ' + sh_out + '% from training data' )
last_amount = test.loc[(test['Country_Region']=='Spain')&(test['Date']=='2020-04-07'),'ConfirmedCases_x']
last_fat = test.loc[(test['Country_Region']=='Spain')&(test['Date']=='2020-04-07'),'Fatalities_x']
i = 0
k = 30
for date in dates:
k = k-1
i = i+1
test.loc[(test['Country_Region']=='Spain')&(test['Date']==date),
'ConfirmedCases_x']=last_amount.values[0] + i*(5000-(100*i))
test.loc[(test['Country_Region']=='Spain')&(test['Date']==date),
'Fatalities_x'] = last_fat.values[0]+i*(800-(10*i))<define_variables>
train_good = train[train['res'] == 1].reset_index(drop=True)
 train_good
last_amount = test.loc[(test['Country_Region']=='China')&(test['Province_State']!='Hubei')&(test['Date']=='2020-04-07'),'ConfirmedCases_x']
last_fat = test.loc[(test['Country_Region']=='China')&(test['Province_State']!='Hubei')&(test['Date']=='2020-04-07'),'Fatalities_x']<feature_engineering>
train_outlier[train_outlier['diff_num']==0]
i = 0
k = 30
for date in dates:
k = k-1
i = i+1
test.loc[(test['Country_Region']=='China')&(test['Province_State']!='Hubei')&(test['Date']==date),
'Fatalities_x']= last_fat.values
test.loc[(test['Country_Region']=='China')&(test['Province_State']!='Hubei')&(test['Date']==date),
'ConfirmedCases_x']= last_amount.values + i<filter>
train_good[train_good['diff_num']==0]
last_amount = test.loc[(test['Country_Region']=='China')&(test['Province_State']=='Hubei')&(test['Date']=='2020-04-07'),'ConfirmedCases_x']
last_fat = test.loc[(test['Country_Region']=='China')&(test['Province_State']=='Hubei')&(test['Date']=='2020-04-07'),'Fatalities_x']<feature_engineering>
train_outlier[train_outlier['share']==1]
k=30
i=0
for date in dates:
k = k-1
i = i+1
test.loc[(test['Country_Region']=='China')&(test['Province_State']=='Hubei')&(test['Date']==date),'ConfirmedCases_x']= last_amount.values[0]
test.loc[(test['Country_Region']=='China')&(test['Province_State']=='Hubei')&(test['Date']==date),'Fatalities_x']= last_fat.values[0] + i<prepare_output>
train_good[train_good['share']==1]
sub = test[['ForecastId','ConfirmedCases_x','Fatalities_x']]<rename_columns>
len(train_outlier[train_outlier["text"].str.find(' ')== -1] )
sub.columns=['ForecastId','ConfirmedCases','Fatalities']<feature_engineering>
len(train_good[train_good["text"].str.find(' ')== -1] )
sub.loc[sub['ConfirmedCases']<0,'ConfirmedCases']=0<feature_engineering>
def subtext_analysis(col, subtext, df1, str1, df2, str2):
result = pd.DataFrame(columns = ['subtext', str1, str2, 'share,%'])
 if(len(df1)> 0)and(len(df2)> 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num1 = len(df1[df1[col].str.find(subtext[i])> -1])
 result.loc[i, str1] = num1
num2 = len(df2[df2[col].str.find(subtext[i])> -1])
 result.loc[i, str2] = num2
result.loc[i,'share,%'] = round(num1*100/num2,1)if num2 != 0 else 0
print('Number of all data is', len(df2))
display(result.sort_values(by=['share,%', str1], ascending=False))
sub.loc[sub['Fatalities']<0, 'Fatalities']=0<save_to_csv>
def subtext_analysis_one_df(col, subtext, df, str):
result = pd.DataFrame(columns = ['subtext', str, 'share of all,%'])
 num_all = len(df)
 if(num_all > 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num = len(df[df[col].str.find(subtext[i])> -1])
 result.loc[i, str] = num
result.loc[i,'share of all,%'] = round(num*100/num_all,1)
 print('Number of all data is', len(df))
display(result.sort_values(by='share of all,%', ascending=False))
sub.to_csv('submission.csv',index=False )<load_from_csv>
subtext_test = ['SAD', 'bullying', 'Uh', 'oh', 'onna', 'fun', 'addicted', 'Power', 'well', 'unhappy', 'funny', 'Tears', 'Fears', 'sleeeeepy', ' ', ',', '?', '!' ,'!!', '!!!', ':/', '...', 'http', '****']
train_dir = ".. /input/train.csv"
test_dir = ".. /input/test.csv"
df = pd.read_csv(train_dir)
 df.info()<categorify>
subtext_analysis("selected_text", subtext_test, train_outlier, 'train_outliers', train, 'train_all' )
labels = df["label"].values.tolist()
labels = np.array(labels)
 n_classes = len(set(labels))
labels = keras.utils.to_categorical(labels )<data_type_conversions>
subtext_analysis("selected_text", subtext_test, train_good, 'train_good', train, 'train_all' )
df_train = df.drop(["label"], axis = 1)
 data = df_train.values.tolist()
data = np.array(data)
 data = data.astype('float32')/255.0<train_model>
subtext_analysis_one_df("selected_text", subtext_test, train, 'test_all' )
print("Training data shape = " + str(data.shape))
print("Training labels shape = " + str(labels.shape))<choose_model_class>
subtext_analysis_one_df("selected_text", subtext_test, test, 'test_all' )
gen_model = Sequential()
gen_model.add(Dense(784, activation = 'relu', input_shape =(784,)))
 gen_model.add(Dense(512, activation = 'relu'))
gen_model.add(Dense(264, activation = 'relu'))
gen_model.add(Dense(10, activation = 'softmax'))
print("STANDARD NEURAL NETWORK MODEL :-")
 gen_model.summary()<choose_model_class>
test['text_chr3'] = test['text'].apply(rep_3chr)
 test.head(10 )
gen_model.compile(loss = 'categorical_crossentropy', optimizer = keras.optimizers.Adadelta() , metrics = ['accuracy'] )<train_model>
train_outlier1 = train_outlier.nsmallest(1000, 'metric', keep='all')
 train_outlier2 = train_outlier.nsmallest(2000, 'metric', keep='all')
 train_outlier3 = train_outlier.nsmallest(3000, 'metric', keep='all')
 train_outlier5 = train_outlier.nsmallest(5000, 'metric', keep='all')
 train_outlier8 = train_outlier.nsmallest(8000, 'metric', keep='all' )
gen_model_hist = gen_model.fit(data, labels, batch_size = 32, epochs = 5, validation_split = 0.1 )<set_options>
subtext_analysis("selected_text", subtext_test, train_outlier1, 'in worst 1000 outliers', train_outlier, 'in all outliers' )
del gen_model, gen_model_hist
gc.collect()<prepare_x_and_y>
subtext_analysis("selected_text", subtext_test, train_outlier2, 'in worst 2000 outliers', train_outlier, 'in all outliers' )
X_train_cnn = data.reshape(len(data), 28, 28, 1 )<choose_model_class>
subtext_analysis("selected_text", subtext_test, train_outlier3, 'in worst 3000 outliers', train_outlier, 'in all outliers' )
cnn_model = Sequential()
cnn_model.add(Conv2D(32, kernel_size = [3,3], activation = 'relu', input_shape =(28,28,1)))
 cnn_model.add(Conv2D(64, kernel_size = [3,3], activation = 'relu'))
cnn_model.add(BatchNormalization())
 cnn_model.add(MaxPool2D(pool_size = [2,2], strides = 2))
cnn_model.add(Conv2D(128, kernel_size = [3,3], activation = 'relu'))
cnn_model.add(MaxPool2D(pool_size = [2,2], strides = 2))
cnn_model.add(Flatten())
 cnn_model.add(Dense(512, activation = 'relu'))
cnn_model.add(Dense(10, activation = 'softmax'))
print("CONVOLUTIONAL NEURAL NETWORK MODEL :-")
 cnn_model.summary()<train_model>
subtext_analysis("selected_text", subtext_test, train_outlier5, 'in worst 5000 outliers', train_outlier, 'in all outliers' )
cnn_model.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
 cnn_model_hist = cnn_model.fit(X_train_cnn, labels, batch_size = 32, epochs = 6, validation_split = 0.1 )<set_options>
subtext_analysis("selected_text", subtext_test, train_outlier8, 'in worst 8000 outliers', train_outlier, 'in all outliers' )
del cnn_model, cnn_model_hist
gc.collect()<choose_model_class>
col_hist = ['sentiment', 'start', 'end', 'start_pred', 'end_pred', 'len_text', 'len_selected_text',
'text_chr3', 'selected_text_chr3', 'selected_text_pred_chr3', 'metric']
<choose_model_class><EOS> 
data = train[['sentiment', 'start', 'end', 'start_pred', 'end_pred', 'len_text', 'len_selected_text', 'diff_num', 'share', 'metric', 'res']].dropna()
data