Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<set_options>
import pandas as pd
import numpy as np
import torch
py.init_notebook_mode(connected=True)
<set_options>
df_train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 df_train.head()
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<load_from_disk>
df_test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 df_test.head()
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )<categorify>
df_train.target.value_counts()
ucf_root = Path('.. /input/ashrae-ucf-spider-and-eda-full-test-labels')
 leak0_df = pd.read_pickle(ucf_root/'site0.pkl')
 leak0_df['meter_reading'] = leak0_df.meter_reading_scraped
leak0_df.drop(['meter_reading_original','meter_reading_scraped'], axis=1, inplace=True)
 leak0_df.fillna(0, inplace=True)
 leak0_df.loc[leak0_df.meter_reading < 0, 'meter_reading'] = 0
leak0_df = leak0_df[leak0_df.timestamp.dt.year > 2016]
print(len(leak0_df))<load_pretrained>
xtrain,xval,ytrain,yval = train_test_split(df_train.index.values, df_train.target.values,
test_size = 0.2, random_state=15, stratify = df_train.target.values)
 print(len(xtrain),len(xval))
ucl_root = Path('.. /usr/lib/ucl_data_leakage_episode_2')
 leak1_df = pd.read_pickle(ucl_root/'site1.pkl')
 leak1_df['meter_reading'] = leak1_df.meter_reading_scraped
leak1_df.drop(['meter_reading_scraped'], axis=1, inplace=True)
 leak1_df.fillna(0, inplace=True)
 leak1_df.loc[leak1_df.meter_reading < 0, 'meter_reading'] = 0
leak1_df = leak1_df[leak1_df.timestamp.dt.year > 2016]
print(len(leak1_df))<concatenate>
df_train['set_type'] = 'nil'*df_train.shape[0]
df_train.loc[xtrain, 'set_type'] = 'train'
df_train.loc[xval, 'set_type'] = 'val'
df_train.head(10 )
if use_ucf:
if del_2016:
print('delete all buildings site0 in 2016')
 bids = leak_df.building_id.unique()
train_df = train_df[train_df.building_id.isin(bids)== False]
leak0_df = leak0_df[leak0_df.timestamp.dt.year.isin(ucf_year)]
leak1_df = leak1_df[leak1_df.timestamp.dt.year.isin(ucf_year)]
train_df = pd.concat([train_df, leak0_df, leak1_df])
 train_df.reset_index(inplace=True)
 weather_train_df = pd.concat([weather_train_df, weather_test_df])
 weather_train_df.reset_index(inplace=True )<drop_column>
df_train.groupby(['target', 'set_type'] ).count()
del weather_test_df, leak0_df, leak1_df
gc.collect()<feature_engineering>
tokenizer= BertTokenizer.from_pretrained('bert-base-uncased',
do_lower_case=True )
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )<filter>
encoded_train = tokenizer.batch_encode_plus(
 df_train[df_train.set_type=='train'].text.values,
add_special_tokens=True,
return_attention_masks=True,
pad_to_max_length=True,
max_length=256,
return_tensors='pt' 
)
 encoded_val = tokenizer.batch_encode_plus(
 df_train[df_train.set_type=='val'].text.values,
add_special_tokens=True,
return_attention_masks=True,
pad_to_max_length=True,
max_length=256,
return_tensors='pt' 
)
 encoded_test = tokenizer.batch_encode_plus(
 df_test.text.values,
add_special_tokens=True,
return_attention_masks=True,
pad_to_max_length=True,
max_length=256,
return_tensors='pt' 
)
 input_ids_train = encoded_train['input_ids']
attention_masks_train = encoded_train['attention_mask']
labels_train = torch.tensor(df_train[df_train.set_type=='train'].target.values)
 input_ids_val = encoded_val['input_ids']
attention_masks_val = encoded_val['attention_mask']
labels_val = torch.tensor(df_train[df_train.set_type=='val'].target.values)
 input_ids_test = encoded_test['input_ids']
attention_masks_test = encoded_test['attention_mask']
building_meta_df[building_meta_df.site_id == 0]<filter>
dataset_train = TensorDataset(input_ids_train, attention_masks_train, labels_train)
 dataset_val = TensorDataset(input_ids_val, attention_masks_val, labels_val)
 dataset_test = TensorDataset(input_ids_test, attention_masks_test )
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )<feature_engineering>
model = BertForSequenceClassification.from_pretrained('bert-base-uncased',
num_labels = 2,
output_attentions = False,
output_hidden_states = False
)
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_localtime(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )<feature_engineering>
dataloader_train = DataLoader(
 dataset_train,
sampler= RandomSampler(dataset_train),
batch_size=32 
)
 dataloader_val = DataLoader(
 dataset_val,
sampler = SequentialSampler(dataset_val),
batch_size=32 
)
 dataloader_test = DataLoader(
 dataset_test,
sampler = SequentialSampler(dataset_test),
batch_size=32
)
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek 
<categorify>
optimizer = AdamW(model.parameters() ,
lr=1e-5,
eps=1e-8)
 epochs = 4
scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=len(dataloader_train)*epochs
)
preprocess(train_df )<sort_values>
def f1_score_func(preds, labels):
preds_flat = np.argmax(preds, axis =1 ).flatten()
labels_flat = labels.flatten()
return f1_score(labels_flat, preds_flat, average='weighted')
 def accuracy_per_class(preds, labels):
preds_flat = np.argmax(preds, axis =1 ).flatten()
labels_flat = labels.flatten()
for label in np.unique(labels_flat):
y_preds = preds_flat[labels_flat==label]
y_true = labels_flat[labels_flat==label]
print(f'Class: {label}')
 print(f'Accuracy: {len(y_preds[y_preds==label])}/{len(y_true)}' )
if use_ucf and use_sort:
train_df = train_df.sort_values('month')
 train_df = train_df.reset_index()<categorify>
seed_val = 10
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 device = 'cuda' if torch.cuda.is_available() else 'cpu'
model.to(device)
 print(device)
df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_median = df_group.median().astype(np.float16)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 del df_group<count_missing_values>
def evaluate(dataloader_val):
model.eval()
loss_val_total = 0
predictions, true_vals = [], []
for batch in dataloader_val:
batch = tuple(b.to(device)for b in batch)
 inputs = {'input_ids': batch[0],
'attention_mask': batch[1],
'labels': batch[2],
}
with torch.no_grad() :
outputs = model(**inputs)
 loss = outputs[0]
logits = outputs[1]
loss_val_total += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = inputs['labels'].cpu().numpy()
predictions.append(logits)
 true_vals.append(label_ids)
 loss_val_avg = loss_val_total/len(dataloader_val)
 predictions = np.concatenate(predictions, axis=0)
 true_vals = np.concatenate(true_vals, axis=0)
 return loss_val_avg, predictions, true_vals
weather_train_df.isna().sum()<groupby>
for epoch in tqdm(range(1, epochs+1)) :
model.train()
training_loss=0
progress_bar = tqdm(dataloader_train, desc='Epoch {:1d}'.format(epoch),
leave=False,
disable=False 
)
 for batch in progress_bar:
model.zero_grad()
batch = tuple(b.to(device)for b in batch)
 inputs = {
'input_ids': batch[0],
'attention_mask':batch[1],
'labels':batch[2]
}
outputs = model(**inputs)
 loss=outputs[0]
training_loss+= loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item() /len(batch)) })
 tqdm.write(f'
Epoch {epoch}')
 loss_train_avg = training_loss/len(dataloader_train)
 tqdm.write(f'Training loss: {loss_train_avg}')
 val_loss, predictions, true_vals = evaluate(dataloader_val)
 val_f1 = f1_score_func(predictions, true_vals)
 tqdm.write(f'Validation Loss: {val_loss}')
 tqdm.write(f'F1 score: {val_f1}')
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )<groupby>
_, predictions, true_vals = evaluate(dataloader_val )
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))<groupby>
accuracy_per_class(predictions, true_vals )
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )<data_type_conversions>
model.eval()
predictions=[]
for batch in dataloader_test:
batch = tuple(b.to(device)for b in batch)
 inputs = {'input_ids': batch[0],
'attention_mask': batch[1]
}
with torch.no_grad() :
outputs = model(**inputs)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(np.argmax(logits,axis=1))
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]<feature_engineering>
prediction = list(chain.from_iterable(predictions))
set_localtime(weather_train_df )<categorify>
sub= pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 sub.head()
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()<feature_engineering>
sub['target']=prediction
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )<define_variables>
sub.to_csv('submission.csv', index=False )
<prepare_x_and_y><EOS> 
sub.to_csv('submission.csv', index=False )