Kaggle competition: Bike Sharing Demand
<SOS> metric: RMSLE Kaggle data source: bike-sharing-demand<load_from_csv>
%matplotlib inline
pd.set_option("display.max_columns", 100)
 def fxn() :
warnings.warn("deprecated", DeprecationWarning)
 with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 fxn()
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 sentences = train.text.values
labels = train.target.values<categorify>
data=pd.read_csv('.. /input/bike-sharing-demand/train.csv')
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 labels = torch.tensor(labels)
 print('Original: ', sentences[0])
 print('Token IDs:', input_ids[0] )<load_pretrained>
data=data.rename(columns={col_names[5]: "Temperature(C)",col_names[6]:"Dew Temperature(C)",col_names[7]: "humidity(%)",
col_names[8]:"Wind Speed(m/s)",col_names[11]:"Number of rented bikes"
} )
batch_size = 16
def get_data_loaders(dataset,train_indexes,val_indexes):
train_tensor = Subset(dataset,train_indexes)
 val_tensor = Subset(dataset,val_indexes)
 train_dataloader = DataLoader(
 train_tensor,
sampler = RandomSampler(train_tensor),
batch_size = batch_size 
)
 val_dataloader = DataLoader(
 val_tensor,
sampler = SequentialSampler(val_tensor),
batch_size = batch_size 
)
 return train_dataloader,val_dataloader 
<create_dataframe>
data.isna().sum()
dataset = TensorDataset(input_ids, attention_masks, labels )<load_from_csv>
def convert_date(df):
df['datetime'] = pd.to_datetime(df['datetime'])
 df['month'] = df['datetime'].dt.month
df['hour'] = df['datetime'].dt.hour
df['weekday'] = df['datetime'].dt.dayofweek
df["day"]=df["datetime"].dt.day
df["year"]=df["datetime"].dt.year
df['dayofweek'] = df['datetime'].dt.dayofweek
df['month_start'] = df['datetime'].dt.is_month_start
df['woy'] = df['datetime'].dt.isocalendar().week.astype(int)
 return df
df = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 sentences = df.text.values
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 batch_size = 16
prediction_data = TensorDataset(input_ids, attention_masks)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size )<load_pretrained>
data=convert_date(data )
def get_bert_model() :
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()
return model<compute_test_metric>
df1VizSeasons=pd.DataFrame(data.groupby('season' ).sum() ['Number of rented bikes'].sort_values(ascending=False))
df1VizSeasons.style.background_gradient(cmap=sns.light_palette("red", as_cmap=True))
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )<categorify>
df1VizDays=pd.DataFrame(data.groupby('dayofweek' ).sum() ['Number of rented bikes'].sort_values(ascending=False))
df1VizDays.style.background_gradient(cmap=sns.light_palette("red", as_cmap=True))
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded)) 
<set_options>
def drop_cols(df):
df=df.drop(['datetime','casual','registered'],axis=1)
 return df
seed_val = 1000
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val )<choose_model_class>
data=drop_cols(data )
total_folds = 12
current_fold = 0
all_folds_preds = []
epochs = 1
fold=StratifiedKFold(n_splits=total_folds, shuffle=True, random_state=1000)
 training_stats = []<train_model>
features_plot=['Number of rented bikes', 'Temperature(C)', 'humidity(%)',
'Wind Speed(m/s)', 'Dew Temperature(C)',
]
total_t0 = time.time()
for train_index, test_index in fold.split(train,train['target']):
model = get_bert_model()
optimizer = AdamW(model.parameters() ,lr = 5e-5,eps = 1e-8)
 current_fold = current_fold+1
train_dataloader,validation_dataloader = get_data_loaders(dataset,train_index,test_index)
 print("")
 print('================= Fold {:} / {:} ================='.format(current_fold,total_folds))
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_time = format_time(time.time() - t0)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
total_f1_score = 0
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in validation_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() : 
(loss, logits)= model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
total_eval_accuracy += flat_accuracy(logits, label_ids)
 total_f1_score += f1_score(np.argmax(logits,axis=1),label_ids)
 avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
 print(" Accuracy: {0:.2f}".format(avg_val_accuracy))
avg_f1_score = total_f1_score / len(validation_dataloader)
 print(" F1_score: {0:.2f}".format(avg_f1_score))
avg_val_loss = total_eval_loss / len(validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
training_stats.append(
 {
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid.Loss': avg_val_loss,
'Valid.Accur.': avg_val_accuracy,
'f1_score' : avg_f1_score,
'Training Time': training_time,
'Validation Time': validation_time,
'fold' : current_fold
} 
)
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)))
 print("")
 print("now predicting for this fold")
 model.eval()
predictions = []
for batch in prediction_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 stack = np.vstack(predictions)
 final_preds = F.softmax(torch.from_numpy(stack)) [:,1].numpy()
all_folds_preds.append(final_preds)
 print("Completed")
<create_dataframe>
def find_skewed_boundaries(df, variable, distance):
IQR = df[variable].quantile(0.75)- df[variable].quantile(0.25)
 lower_boundary = df[variable].quantile(0.25)-(IQR * distance)
 upper_boundary = df[variable].quantile(0.75)+(IQR * distance)
 return upper_boundary, lower_boundary
pd.set_option('precision', 2)
 df_stats = pd.DataFrame(data=training_stats)
 df_stats = df_stats.set_index('fold')
 df_stats<feature_engineering>
def additional(df):
df['ideal'] = df[['Temperature(C)', 'Wind Speed(m/s)']].apply(lambda x:(0, 1)[x['Temperature(C)'] > 27 and x['Wind Speed(m/s)'] < 30], axis = 1)
 df['sticky'] = df[['humidity(%)', 'workingday']].apply(lambda x:(0, 1)[x['workingday'] == 1 and x['humidity(%)'] >= 60], axis = 1)
 df["windchill"]=13.12+0.6215*df['Temperature(C)']-11.37*(df['Wind Speed(m/s)']*3.6)**0.16+0.3965*df['Temperature(C)'] \
*(df['Wind Speed(m/s)']*3.6)**0.16
df['newweather']=17.3+df['Temperature(C)']-0.11*df['humidity(%)']+0.34*df['Wind Speed(m/s)']
df['hour_sin'] = np.sin(2 * np.pi * df['hour']/23.0)
 df['hour_cos'] = np.cos(2 * np.pi * df['hour']/23.0)
 df['month_sin'] = np.sin(2 * np.pi * df['month']/12)
 df['month_cos'] = np.cos(2 * np.pi * df['month']/12)
 df['log_wind']=np.log1p(df['Wind Speed(m/s)'])
 df['log_hum']=np.log1p(df['humidity(%)'])
 return df
to_submit =np.mean(all_folds_preds,0 )<prepare_output>
cond1 = data['workingday']== 'Yes'
cond2 = data['hour'].isin([7,8,9,17,18,19])
 cond3 = data['hour'].isin([10,11,12,13,14,15,16])
 cond4 = data['hour'].isin([0,1,2,3,4,5,6,20,21,22,23])
 conditions =[~cond1,cond1&cond2,cond1&cond3,cond1&cond4]
vals = ['No','High','Medium','Low']
data['Map demand'] = np.select(conditions,vals)
sub=pd.DataFrame()
sub['id'] = df['id']
sub['target'] = to_submit
sub['target'] = sub['target'].apply(lambda x: 1 if x>0.5 else 0)
 sub.head()<save_to_csv>
data=additional(data )
sub.to_csv('bert_base_12_2e-5_64.csv',index=False )<install_modules>
cat_features = data.columns[data.dtypes=='object']
cat_features=list(cat_features)
 cat_features
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null
!python pytorch-xla-env-setup.py --version 20200420 --apt-packages libomp5 libopenblas-dev > /dev/null
!pip install transformers==2.5.1 > /dev/null
!pip install pandarallel > /dev/null
!pip install catalyst==20.4.2 > /dev/null<set_options>
def count_category(name):
print(name)
 print(data[name].value_counts())
 print("-------------------------------------------------")
 for i in cat_features:
count_category(i )
os.environ['XLA_USE_BF16'] = "1"
tqdm.pandas()
nltk.download('punkt')
 pandarallel.initialize(nb_workers=4, progress_bar=False )<define_variables>
count_bikes_by_hour = data.groupby("hour")["Number of rented bikes"].sum()
print(count_bikes_by_hour )
SEED = 42
MAX_LENGTH = 64
BACKBONE_PATH = 'roberta-base'
ROOT_PATH = f'.. '
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<string_transform>
count_bikes_by_Holiday = data.groupby("holiday")["Number of rented bikes"].sum()
print(count_bikes_by_Holiday )
def get_sentences(text):
sentences = sent_tokenize(text)
 return ' '.join(sentences)
 def clean_text(text):
text = str(text)
 text = re.sub(r'[0-9"]', '', text)
 text = re.sub(r'
text = re.sub(r'@[\S]+\b', '', text)
 text = re.sub(r'https?\S+', '', text)
 text = re.sub(r'\s+', ' ', text)
 text = get_sentences(text)
 return text.strip()
class NLPTransform(BasicTransform):

 @property
def targets(self):
return {"data": self.apply}
def update_params(self, params, **kwargs):
if hasattr(self, "interpolation"):
params["interpolation"] = self.interpolation
if hasattr(self, "fill_value"):
params["fill_value"] = self.fill_value
return params
def get_sentences(self, text):
return sent_tokenize(text)
 class ShuffleSentencesTransform(NLPTransform):

 def __init__(self, always_apply=False, p=0.5):
super(ShuffleSentencesTransform, self ).__init__(always_apply, p)
 def apply(self, data, **params):
text = data
sentences = self.get_sentences(text)
 random.shuffle(sentences)
 return ' '.join(sentences)
 class ExcludeNumbersTransform(NLPTransform):

 def __init__(self, always_apply=False, p=0.5):
super(ExcludeNumbersTransform, self ).__init__(always_apply, p)
 def apply(self, data, **params):
text = data
text = re.sub(r'[0-9]', '', text)
 text = re.sub(r'\s+', ' ', text)
 return text
class ExcludeHashtagsTransform(NLPTransform):
Exclude @users Exclude urls<feature_engineering>
count_bikes_by_funcday = data.groupby("workingday")["Number of rented bikes"].sum()
print(count_bikes_by_funcday )
def get_train_transforms() :
return albumentations.Compose([
ExcludeUsersMentionedTransform(p=0.95),
ExcludeUrlsTransform(p=0.95),
ExcludeNumbersTransform(p=0.95),
ExcludeHashtagsTransform(p=0.95),
], p=1.0)
 train_transforms = get_train_transforms()
tokenizer = RobertaTokenizer.from_pretrained(BACKBONE_PATH)
 shuffle_transforms = ShuffleSentencesTransform(always_apply=True )<categorify>
count_bikes_by_Season = data.groupby("season")["Number of rented bikes"].sum()
print(count_bikes_by_Season )
def onehot(size, target):
vec = torch.zeros(size, dtype=torch.float32)
 vec[target] = 1.
 return vec
class DatasetRetriever(Dataset):
def __init__(self, labels_or_ids, texts, use_train_transforms=False, test=False):
self.test = test
self.labels_or_ids = labels_or_ids
self.texts = texts
self.use_train_transforms = use_train_transforms
def get_tokens(self, text):
encoded = tokenizer.encode_plus(
 text,
add_special_tokens=True,
max_length=MAX_LENGTH,
pad_to_max_length=True 
)
 return encoded['input_ids'], encoded['attention_mask']
def __len__(self):
return self.texts.shape[0]
def __getitem__(self, idx):
text = self.texts[idx]
if self.test is False:
label = self.labels_or_ids[idx]
target = onehot(2, label)
 if self.use_train_transforms:
text = train_transforms(data=(text)) ['data']
tokens, attention_mask = self.get_tokens(str(text))
token_length = sum(attention_mask)
 if token_length > 0.8*MAX_LENGTH:
text = shuffle_transforms(data=(text)) ['data']
else:
tokens, attention_mask = torch.tensor(tokens), torch.tensor(attention_mask)
 return target, tokens, attention_mask
tokens, attention_mask = self.get_tokens(str(text))
tokens, attention_mask = torch.tensor(tokens), torch.tensor(attention_mask)
 if self.test is False:
return target, tokens, attention_mask
return self.labels_or_ids[idx], tokens, attention_mask
def get_labels(self):
return list(self.labels_or_ids.astype(str))<load_from_csv>
def rolling_mean(df,cols,window=3):
for col in cols:
new_col = col+'_rolled'
df[new_col] = df[col].rolling(window=window ).mean()
return df
df = pd.read_csv(f'{ROOT_PATH}/input/nlp-getting-started/train.csv')
 X = df['text'].values
y = df['target'].values
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.25, random_state=42, stratify=y)
 df_train = pd.DataFrame(
 {'target': y_train,
'text': X_train})
 df_val = pd.DataFrame(
 {'target': y_val,
'text': X_val})
 del df
gc.collect() ;<feature_engineering>
data = rolling_mean(data,['Temperature(C)','Wind Speed(m/s)',
'humidity(%)','weather'])
validation_tune_dataset = DatasetRetriever(
 labels_or_ids=df_val['target'].values,
texts=df_val['text'].values,
use_train_transforms=True, 
)
 df_val['text'] = df_val.apply(lambda x: clean_text(x['text']), axis=1)
 validation_dataset = DatasetRetriever(
 labels_or_ids=df_val['target'].values,
texts=df_val['text'].values,
use_train_transforms=False, 
)
 del df_val
gc.collect() ;
for targets, tokens, attention_masks in validation_dataset:
break
print(targets)
 print(tokens.shape)
 print(attention_masks.shape )<load_from_csv>
def shifting(df,cols):
for col in cols:
new_col = col+'_shifted'
df[new_col] = df[col].shift(periods=-1)
 return df
data = shifting(data,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'])
 def shifting_1(df,cols):
for col in cols:
new_col = col+'_shifted1'
df[new_col] = df[col].shift(periods=-2)
 return df
data = shifting_1(data,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'])
 def shifting_2(df,cols):
for col in cols:
new_col = col+'_shifted2'
df[new_col] = df[col].shift(periods=-3)
 return df
data = shifting_2(data,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'] )
df_test = pd.read_csv(f'{ROOT_PATH}/input/nlp-getting-started/test.csv', index_col='id')
 df_test['text'] = df_test.parallel_apply(lambda x: clean_text(x['text']), axis=1)
 test_dataset = DatasetRetriever(
 labels_or_ids=df_test.index.values,
texts=df_test['text'].values,
use_train_transforms=False,
test=True 
)
 del df_test
gc.collect() ;
for ids, tokens, attention_masks in test_dataset:
break
print(ids)
 print(tokens.shape)
 print(attention_masks.shape )<compute_train_metric>
data=data.fillna(data.mean() )
class RocAucMeter(object):
def __init__(self):
self.reset()
def reset(self):
self.y_true = np.array([0,1])
 self.y_pred = np.array([0.5,0.5])
 self.score = 0
def update(self, y_true, y_pred):
y_true = y_true.cpu().numpy().argmax(axis=1)
 y_pred = nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,1]
self.y_true = np.hstack(( self.y_true, y_true))
self.y_pred = np.hstack(( self.y_pred, y_pred))
self.score = sklearn.metrics.roc_auc_score(self.y_true, self.y_pred, labels=np.array([0, 1]))
@property
def avg(self):
return self.score
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<normalization>
def conv_cat_features(df):
df['holiday'].replace({"No Holiday":1, "Holiday":0}, inplace=True)
 df['workingday'].replace({"Yes":1, "No":0}, inplace=True)
 data_cat_transformed = pd.get_dummies(df, drop_first=True)
 return data_cat_transformed
class LabelSmoothing(nn.Module):
def __init__(self, smoothing = 0.1):
super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
def forward(self, x, target):
if self.training:
x = x.float()
target = target.float()
logprobs = torch.nn.functional.log_softmax(x, dim = -1)
 nll_loss = -logprobs * target
nll_loss = nll_loss.sum(-1)
 smooth_loss = -logprobs.mean(dim=-1)
 loss = self.confidence * nll_loss + self.smoothing * smooth_loss
return loss.mean()
else:
return torch.nn.functional.cross_entropy(x, target )<set_options>
data_cat_transformed=conv_cat_features(data)
 print("{} total features after one-hot encoding.".format(len(data_cat_transformed.columns)))
 print(data_cat_transformed.columns )
warnings.filterwarnings("ignore")
 class TPUFitter:
def __init__(self, model, device, config):
if not os.path.exists('node_submissions'):
os.makedirs('node_submissions')
 self.config = config
self.epoch = 0
self.log_path = 'log.txt'
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = AdamW(optimizer_grouped_parameters, lr=config.lr*xm.xrt_world_size())
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.criterion = config.criterion
xm.master_print(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
para_loader = pl.ParallelLoader(train_loader, [self.device])
 losses, final_scores = self.train_one_epoch(para_loader.per_device_loader(self.device))
self.log(f'[RESULT]: Train.Epoch: {self.epoch}, loss: {losses.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 t = time.time()
para_loader = pl.ParallelLoader(validation_loader, [self.device])
 losses, final_scores = self.validation(para_loader.per_device_loader(self.device))
self.log(f'[RESULT]: Validation.Epoch: {self.epoch}, loss: {losses.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 if self.config.validation_scheduler:
self.scheduler.step(metrics=final_scores.avg)
 self.epoch += 1
def run_tuning_and_inference(self, test_loader, validation_tune_loader):
for e in range(2):
self.optimizer.param_groups[0]['lr'] = self.config.lr*xm.xrt_world_size()
para_loader = pl.ParallelLoader(validation_tune_loader, [self.device])
 losses, final_scores = self.train_one_epoch(para_loader.per_device_loader(self.device))
para_loader = pl.ParallelLoader(test_loader, [self.device])
 self.run_inference(para_loader.per_device_loader(self.device))
def validation(self, val_loader):
self.model.eval()
losses = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(targets, inputs, attention_masks)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
xm.master_print(
 f'Valid Step {step}, loss: ' + \
f'{losses.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}' 
)
 with torch.no_grad() :
inputs = inputs.to(self.device, dtype=torch.long)
 attention_masks = attention_masks.to(self.device, dtype=torch.long)
 targets = targets.to(self.device, dtype=torch.float)
 outputs = self.model(inputs, attention_masks)
 loss = self.criterion(outputs, targets)
 batch_size = inputs.size(0)
 final_scores.update(targets, outputs)
 losses.update(loss.detach().item() , batch_size)
 return losses, final_scores
def train_one_epoch(self, train_loader):
self.model.train()
losses = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(targets, inputs, attention_masks)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
self.log(
 f'Train Step {step}, loss: ' + \
f'{losses.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}' 
)
 inputs = inputs.to(self.device, dtype=torch.long)
 attention_masks = attention_masks.to(self.device, dtype=torch.long)
 targets = targets.to(self.device, dtype=torch.float)
 self.optimizer.zero_grad()
outputs = self.model(inputs, attention_masks)
 loss = self.criterion(outputs, targets)
 batch_size = inputs.size(0)
 final_scores.update(targets, outputs)
 losses.update(loss.detach().item() , batch_size)
 loss.backward()
xm.optimizer_step(self.optimizer)
 if self.config.step_scheduler:
self.scheduler.step()
self.model.eval()
self.save('last-checkpoint.bin')
 return losses, final_scores
def run_inference(self, test_loader):
self.model.eval()
result = {'id': [], 'target': []}
t = time.time()
for step,(ids, inputs, attention_masks)in enumerate(test_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
xm.master_print(f'Prediction Step {step}, time: {(time.time() - t):.5f}')
 with torch.no_grad() :
inputs = inputs.to(self.device, dtype=torch.long)
 attention_masks = attention_masks.to(self.device, dtype=torch.long)
 outputs = self.model(inputs, attention_masks)
 toxics = nn.functional.softmax(outputs, dim=1 ).data.cpu().numpy() [:,1]
result['id'].extend(ids.cpu().numpy())
 result['target'].extend(toxics)
 result = pd.DataFrame(result)
 node_count = len(glob('node_submissions/*.csv'))
result.to_csv(f'node_submissions/submission_{node_count}_{datetime.utcnow().microsecond}_{random.random() }.csv', index=False)
 def save(self, path):
xm.save(self.model.state_dict() , path)
 def log(self, message):
if self.config.verbose:
xm.master_print(message)
 with open(self.log_path, 'a+')as logger:
xm.master_print(f'{message}', logger )<choose_model_class>
X = data_cat_transformed.drop('Number of rented bikes', axis=1)
 y = data_cat_transformed['Number of rented bikes']
def custom_train_valid_split(data, cutoff_day=15):
train = data[data['day'] <= cutoff_day]
valid = data[data['day'] > cutoff_day]
return train,valid
train,valid = custom_train_valid_split(data, cutoff_day=15)
 train_idx = train.index
valid_idx = valid.index
myCViterator = []
myCViterator.append(( train_idx, valid_idx))
class NNModel(nn.Module):
def __init__(self):
super(NNModel, self ).__init__()
self.backbone = RobertaModel.from_pretrained(BACKBONE_PATH)
 self.dropout = nn.Dropout(0.3)
 self.linear = nn.Linear(
 in_features=self.backbone.pooler.dense.out_features*2,
out_features=2, 
)
 def forward(self, input_ids, attention_masks):
bs, seq_length = input_ids.shape
seq_x, _ = self.backbone(input_ids=input_ids, attention_mask=attention_masks)
 apool = torch.mean(seq_x, 1)
 mpool, _ = torch.max(seq_x, 1)
 x = torch.cat(( apool, mpool), 1)
 x = self.dropout(x)
 return self.linear(x )<choose_model_class>
def RMSLE(y_train,y_pred):
return np.sqrt(mean_squared_log_error(y_train, y_pred))
net = NNModel()<init_hyperparams>
test_df=pd.read_csv('.. /input/bike-sharing-demand/test.csv')
 test_df_copy=test_df.copy()
class TrainGlobalConfig:

 num_workers = 0
batch_size = 16
n_epochs = 3
lr = 0.5 * 1e-5
fold_number = 0
verbose = True
verbose_step = 5
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='max',
factor=0.7,
patience=0,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08 
)
 criterion = LabelSmoothing() 
<define_search_model>
test_df=test_df.rename(columns={col_names[5]: "Temperature(C)",col_names[6]:"Dew Temperature(C)",col_names[7]: "humidity(%)",
col_names[8]:"Wind Speed(m/s)"
} )
def _mp_fn(rank, flags):
device = xm.xla_device()
net.to(device)
 train_sampler = DistributedSamplerWrapper(
 sampler=BalanceClassSampler(labels=train_dataset.get_labels() , mode="downsampling"),
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True 
)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=train_sampler,
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers, 
)
 validation_sampler = torch.utils.data.distributed.DistributedSampler(
 validation_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 validation_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=validation_sampler,
pin_memory=False,
drop_last=False,
num_workers=TrainGlobalConfig.num_workers 
)
 validation_tune_sampler = torch.utils.data.distributed.DistributedSampler(
 validation_tune_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True 
)
 validation_tune_loader = torch.utils.data.DataLoader(
 validation_tune_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=validation_tune_sampler,
pin_memory=False,
drop_last=False,
num_workers=TrainGlobalConfig.num_workers 
)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=test_sampler,
pin_memory=False,
drop_last=False,
num_workers=TrainGlobalConfig.num_workers 
)
 if rank == 0:
time.sleep(1)
 fitter = TPUFitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, validation_loader)
 fitter.run_tuning_and_inference(test_loader, validation_tune_loader )<feature_engineering>
test_df=convert_date(test_df )
submission['target'] = submission['target'].apply(lambda x: round(x))<save_to_csv>
cond1 = test_df['workingday']=='Yes'
cond2 = test_df['hour'].isin([7,8,9,17,18,19])
 cond3 = test_df['hour'].isin([10,11,12,13,14,15,16])
 cond4 = test_df['hour'].isin([0,1,2,3,4,5,6,20,21,22,23])
 conditions =[~cond1,cond1&cond2,cond1&cond3,cond1&cond4]
vals = ['No','High','Medium','Low']
test_df['Map demand'] = np.select(conditions,vals )
submission.to_csv('submission.csv' )<install_modules>
test_df=test_df.drop(['datetime',],axis=1)
!pip install transformers<import_modules>
test_df=additional(test_df )
nltk.download('stopwords')
 set(stopwords.words('english'))
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<set_options>
test_df = rolling_mean(test_df,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'])
 test_df = shifting(test_df,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'])
 test_df = shifting_1(test_df,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'])
 test_df = shifting_2(test_df,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'])
 test_df=test_df.fillna(test_df.mean() )
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )<load_from_csv>
test_cat_transformed=conv_cat_features(test_df )
df_train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 df_test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 df_submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
<count_values>
df_train.target.value_counts(normalize=True )<feature_engineering>
alphanumeric = lambda x: re.sub('\w*\d\w*', ' ', x)
 punc_lower = lambda x: re.sub('[%s]' % re.escape(string.punctuation), ' ', x.lower())
 ' in the string and replace it with a space
remove_n = lambda x: re.sub("
", " ", x)
 remove_non_ascii = lambda x: re.sub(r'[^\x00-\x7f]',r' ', x)
 df_train['text'] = df_train['text'].map(alphanumeric ).map(punc_lower ).map(remove_n ).map(remove_non_ascii)
 df_test['text'] = df_test['text'].map(alphanumeric ).map(punc_lower ).map(remove_n ).map(remove_non_ascii)
 df_train['text'][3]<define_variables>
X_train=X.copy()
X_test=test_cat_transformed.copy()
sentences = df_train.text.values
labels = df_train.target.values
sentences_test = df_test.text.values
given_labels = np.array(df_submission['target'])
 tensor_labels = torch.tensor(given_labels)
<load_pretrained>
X_train_cat = X_train.copy()
X_test_cat = X_test.copy()
print('Loading BERT tokenizer...')
 tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True )<string_transform>
xgb_v = XGBRegressor(
 objective='count:poisson',random_state=42)
 xgb= XGBRegressor(
 objective='count:poisson',random_state=42,
subsample= 0.83, n_estimators= 3000,
min_child_weight= 16, max_depth= 5,
learning_rate= 0.055, gamma= 1, colsample_bytree= 0.5)
 catboost_v=CatBoostRegressor(objective='Poisson',iterations=4000,l2_leaf_reg=1,learning_rate=0.054 , silent = True)
 catboosttrial=CatBoostRegressor(objective='Poisson',iterations=4000,l2_leaf_reg=1,learning_rate=0.054,silent=True)
 xgb_models={
"XGB_vanella":xgb_v,
"XGB_mod" : xgb }
cat_models ={
"Catbbost_mod" :catboosttrial,
"Catboost_Vanilla" :catboost_v}
print(' Original: ', sentences[100])
 print('Tokenized: ', tokenizer.tokenize(sentences[100]))
print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[100])) )<categorify>
columns = ['Classifier', 'RMSLE']
df_scores_cat = pd.DataFrame(columns=columns)
 for name, model in cat_models.items() :
model.fit(X_train_cat, y)
 Y_predicted = model.predict(X_train_cat)
 rmsle = np.sqrt(mean_squared_log_error(y, Y_predicted))
df_row = pd.DataFrame([[name, rmsle]], columns=columns)
 df_scores_cat = df_scores_cat.append(df_row, ignore_index=True )
def tocknize(text_input):
input_ids = []
for sent in text_input:
encoded_sent = tokenizer.encode(
 sent,
add_special_tokens = True, 
)
 input_ids.append(encoded_sent)
 return input_ids<init_hyperparams>
columns = ['Classifier', 'RMSLE']
df_scores_xgb = pd.DataFrame(columns=columns)
 for name, model in xgb_models.items() :
model.fit(X_train, y)
 Y_predicted = model.predict(X_train)
 rmsle = np.sqrt(mean_squared_log_error(y, Y_predicted))
df_row = pd.DataFrame([[name, rmsle]], columns=columns)
 df_scores_xgb = df_scores_xgb.append(df_row, ignore_index=True )
MAX_LEN = 64
epochs = 2
BATCH_SIZE = 32
random_state=42
seed_val = 2018
steps = 50<categorify>
df_scores_cat.sort_values(by=['RMSLE'] )
def padding_seq(input_token):
out_seq = pad_sequences(input_token,
maxlen=MAX_LEN,
dtype="long",
truncating="post",
padding="post")
 return out_seq
def mask_atten(input_seq):
attention_masks = []
for seq in input_seq:
out_mask = [float(i>0)for i in seq]
attention_masks.append(out_mask)
 return attention_masks 
<categorify>
df_scores_xgb.sort_values(by=['RMSLE'] )
train_token = tocknize(sentences)
 test_token = tocknize(sentences_test)
 train_token = padding_seq(train_token)
 test_token = padding_seq(test_token)
 train_mask = mask_atten(train_token)
 test_mask = mask_atten(test_token )<split>
y_cat_pred=catboosttrial.predict(X_test_cat )
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(train_token, labels,
random_state=random_state, test_size=0.1)
 train_masks, validation_masks, _, _ = train_test_split(train_mask, labels,
random_state=random_state, test_size=0.1 )<data_type_conversions>
y_xgb_pred=xgb.predict(X_test )
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels)
 validation_labels = torch.tensor(validation_labels)
 train_masks = torch.tensor(train_masks)
 validation_masks = torch.tensor(validation_masks)
 tensor_input = torch.tensor(test_token)
 tensor_mask = torch.tensor(test_mask)
<create_dataframe>
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=BATCH_SIZE, shuffle=False)
 prediction_data = TensorDataset(tensor_input, tensor_mask, tensor_labels)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler = prediction_sampler, batch_size = BATCH_SIZE)
<load_pretrained>
test_df_copy['count'] = y_cat_pred.astype('int' )
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()<train_model>
test_df_copy[test_df_copy["workingday"]=='No']
params = list(model.named_parameters())
 print('The BERT model has {:} different named parameters.
 '.format(len(params)))
 print('==== Embedding Layer ====
')
 for p in params[0:5]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== First Transformer ====
')
 for p in params[5:21]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== Output Layer ====
')
 for p in params[-4:]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))<choose_model_class>
test_df_copy[['datetime','count']].to_csv("submissions1.csv",index=False )