Kaggle competition: Bike Sharing Demand
main_sub = pd.read_csv('/kaggle/input/tweet-rake-submission/submission.csv')
 main_sub.to_csv('submission.csv', index=False )<set_options>
train = pd.read_csv('.. /input/bike-sharing-demand/train.csv')
 test = pd.read_csv('.. /input/bike-sharing-demand/test.csv' )
%matplotlib inline
pd.set_option('max_colwidth', 40 )<define_variables>
tt = train.append(test)
 tt.shape
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<feature_engineering>
tt.reset_index().drop('index', axis=1, inplace=True)
 tt.head()
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1<init_hyperparams>
pd.DatetimeIndex(train['datetime'] )
Dropout_new = 0.15
n_split = 5
lr = 3e-5<train_model>
temp = pd.DatetimeIndex(train['datetime'])
 train['year'] = temp.year
train['date'] = temp.date
train['hour'] = temp.hour
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=lr)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model<string_transform>
pd.DatetimeIndex(train.date ).dayofweek
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
train['dayofweek'] = pd.DatetimeIndex(train.date ).dayofweek
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1<define_variables>
pd.to_datetime(train['datetime'] )
%%time
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_end_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=n_split,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting all Train for Outlier analysis...')
 preds_train = padded_model.predict([input_ids,attention_mask,token_type_ids],verbose=DISPLAY)
 preds_start_train += preds_train[0]/skf.n_splits
preds_end_train += preds_train[1]/skf.n_splits
print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<compute_test_metric>
temp = pd.DatetimeIndex(tt['datetime'])
 tt['year'] = temp.year
tt['hour'] = temp.hour
tt = tt[['hour', 'year', 'workingday', 'holiday', 'season', 'weather', 'atemp', 'count']]
tt = pd.get_dummies(tt, columns=['hour'], prefix=['hour'], drop_first=True)
 tt = pd.get_dummies(tt, columns=['year'], prefix=['year'], drop_first=True)
 tt = pd.get_dummies(tt, columns=['season'], prefix=['season'], drop_first=True)
 tt = pd.get_dummies(tt, columns=['weather'], prefix=['weather'], drop_first=True)
 tt.head()
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
print(jac )<string_transform>
new_train = tt.iloc[:10886, :]
new_test = tt.iloc[10886:, :].drop('count', axis=1)
 y = np.log1p(new_train['count'])
 new_train.drop('count', axis=1, inplace=True)
 x = new_train
x.head()
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )<save_to_csv>
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=3 )
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 test.sample(10 )<define_variables>
gbr = GradientBoostingRegressor(random_state=3 )
all = []
start = []
end = []
start_pred = []
end_pred = []
for k in range(input_ids.shape[0]):
a = np.argmax(preds_start_train[k,])
 b = np.argmax(preds_end_train[k,])
 start.append(np.argmax(start_tokens[k]))
end.append(np.argmax(end_tokens[k]))
if a>b:
st = train.loc[k,'text']
start_pred.append(0)
 end_pred.append(len(st))
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 start_pred.append(a)
 end_pred.append(b)
 all.append(st)
 train['start'] = start
train['end'] = end
train['start_pred'] = start_pred
train['end_pred'] = end_pred
train['selected_text_pred'] = all
train.sample(10 )<compute_train_metric>
params = {'learning_rate': [0.1, 0.01, 0.001, 0.5, 0.05],
'n_estimators': [100, 200, 300, 400]}
grid = GridSearchCV(gbr, param_grid=params, cv=5)
 grid.fit(x, y )
def metric_tse(df,col1,col2):
return df.apply(lambda x: jaccard(x[col1],x[col2]),axis=1 )<feature_engineering>
grid.best_params_
train = train.replace({'sentiment': {'negative': -1, 'neutral': 0, 'positive': 1}})
 train['len_text'] = train['text'].str.len()
train['len_selected_text'] = train['selected_text'].str.len()
train['diff_num'] = train['end']-train['start']
train['share'] = train['len_selected_text']/train['len_text']<feature_engineering>
gbr = GradientBoostingRegressor(learning_rate=0.5, n_estimators=400)
 cross_val_score(gbr, x, y, cv=5 ).mean()
train['selected_text_pred'] = train['selected_text_pred'].map(lambda x: x.lstrip(' '))
train['len_selected_text_pred'] = train['selected_text_pred'].str.len()
train['diff_num_pred'] = train['end_pred']-train['start_pred']
train['share_pred'] = train['len_selected_text_pred']/train['len_text']
train['len_equal'] = 0
train.loc[(train['start'] == train['start_pred'])&(train['end'] == train['end_pred']), 'len_equal'] = 1
train['metric'] = metric_tse(train,'selected_text','selected_text_pred')
 train['res'] = 0
train.loc[train['metric'] == 1, 'res'] = 1<string_transform>
gbr.fit(X_train, y_train)
 pre = gbr.predict(X_test)
 mean_squared_error(pre, y_test )
def rep_3chr(text):
chr3 = 0
for word in text.split() :
for c in set(word):
if word.rfind(c+c+c)> -1:
chr3 = 1
return chr3<feature_engineering>
rbf = RandomForestRegressor(n_estimators=222, random_state=50, max_features='sqrt' )
train['text_chr3'] = train['text'].apply(rep_3chr)
 train['selected_text_chr3'] = train['selected_text'].apply(rep_3chr)
 train['selected_text_pred_chr3'] = train['selected_text_pred'].apply(rep_3chr )<filter>
cross_val_score(rbf, x, y, cv=5 ).mean()
<define_variables><EOS> 
gbr.fit(x, y)
 co = gbr.predict(new_test)
 m = []
for i in(np.exp(co)- 1):
n = round(i)
 m.append(n)
 predict = pd.DataFrame({'datetime': test['datetime'], 'count': m})
 predict.to_csv('gbr.csv', index=False )