Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<categorify>
import numpy as np
import pandas as pd
import os
df_train = reduce_mem_usage(df_train, use_float16=True)
 df_test = reduce_mem_usage(df_test, use_float16=True)
 weather_train.timestamp = pd.to_datetime(weather_train.timestamp, format='%Y-%m-%d %H:%M:%S')
 weather_test.timestamp = pd.to_datetime(weather_test.timestamp, format='%Y-%m-%d %H:%M:%S')
 weather_train = reduce_mem_usage(weather_train, use_float16=True)
 weather_test = reduce_mem_usage(weather_test, use_float16=True )<feature_engineering>
import re
import seaborn as sns
import matplotlib.pyplot as plt
from collections import defaultdict, Counter
from sklearn.feature_extraction.text import CountVectorizer
import nltk
from nltk.corpus import stopwords
from wordcloud import WordCloud
from nltk.tokenize import word_tokenize
df_train["hour"] = df_train.timestamp.dt.hour
df_train["weekday"] = df_train.timestamp.dt.weekday
df_test["hour"] = df_test.timestamp.dt.hour
df_test["weekday"] = df_test.timestamp.dt.weekday<merge>
nltk.download('stopwords', quiet=True)
 stopwords = stopwords.words('english')
 sns.set(style="white", font_scale=1.2)
 plt.rcParams["figure.figsize"] = [10,8]
pd.set_option.display_max_columns = 0
pd.set_option.display_max_rows = 0
df_building_meter = df_train.groupby(["building_id", "meter"] ).agg(mean_building_meter=("log_meter_reading", "mean"),
median_building_meter=("log_meter_reading", "median")).reset_index()
df_train = df_train.merge(df_building_meter, on=["building_id", "meter"])
 df_test = df_test.merge(df_building_meter, on=["building_id", "meter"])
 df_building_meter_hour = df_train.groupby(["building_id", "meter", "hour"] ).agg(mean_building_meter=("log_meter_reading", "mean"),
median_building_meter=("log_meter_reading", "median")).reset_index()
df_train = df_train.merge(df_building_meter_hour, on=["building_id", "meter", "hour"])
 df_test = df_test.merge(df_building_meter_hour, on=["building_id", "meter", "hour"] )<data_type_conversions>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv" )
def create_lag_features(df, window):

 feature_cols = ["air_temperature", "cloud_coverage", "dew_temperature", "precip_depth_1_hr"]
df_site = df.groupby("site_id")
 df_rolled = df_site[feature_cols].rolling(window=window, min_periods=0)
 df_mean = df_rolled.mean().reset_index().astype(np.float16)
 df_median = df_rolled.median().reset_index().astype(np.float16)
 df_min = df_rolled.min().reset_index().astype(np.float16)
 df_max = df_rolled.max().reset_index().astype(np.float16)
 df_std = df_rolled.std().reset_index().astype(np.float16)
 df_skew = df_rolled.skew().reset_index().astype(np.float16)
 for feature in feature_cols:
df[f"{feature}_mean_lag{window}"] = df_mean[feature]
df[f"{feature}_median_lag{window}"] = df_median[feature]
df[f"{feature}_min_lag{window}"] = df_min[feature]
df[f"{feature}_max_lag{window}"] = df_max[feature]
df[f"{feature}_std_lag{window}"] = df_std[feature]
df[f"{feature}_skew_lag{window}"] = df_std[feature]
return df<merge>
null_counts = pd.DataFrame({"Num_Null": train.isnull().sum() })
 null_counts["Pct_Null"] = null_counts["Num_Null"] / train.count() * 100
null_counts
weather_train = create_lag_features(weather_train, 18)
 weather_train.drop(["air_temperature", "cloud_coverage", "dew_temperature", "precip_depth_1_hr"], axis=1, inplace=True)
 df_train = df_train.merge(weather_train, on=["site_id", "timestamp"], how="left")
 del weather_train
gc.collect()<define_variables>
len(train["keyword"].value_counts() )
categorical_features = [
"building_id",
"primary_use",
"meter",
"weekday",
"hour"
]
all_features = [col for col in df_train.columns if col not in ["timestamp", "site_id", "meter_reading", "log_meter_reading"]]<split>
disaster_keywords = train.loc[train["target"] == 1]["keyword"].value_counts()
nondisaster_keywords = train.loc[train["target"] == 0]["keyword"].value_counts()
cv = 2
models = {}
cv_scores = {"site_id": [], "cv_score": []}
for site_id in tqdm(range(16), desc="site_id"):
print(cv, "fold CV for site_id:", site_id)
 kf = KFold(n_splits=cv, random_state=seed)
 models[site_id] = []
X_train_site = df_train[df_train.site_id==site_id].reset_index(drop=True)
 y_train_site = X_train_site.log_meter_reading
y_pred_train_site = np.zeros(X_train_site.shape[0])
 score = 0
for fold,(train_index, valid_index)in enumerate(kf.split(X_train_site, y_train_site)) :
X_train, X_valid = X_train_site.loc[train_index, all_features], X_train_site.loc[valid_index, all_features]
y_train, y_valid = y_train_site.iloc[train_index], y_train_site.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train, categorical_feature=categorical_features)
 dvalid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=categorical_features)
 watchlist = [dtrain, dvalid]
params = {"objective": "regression",
"num_leaves": 41,
"learning_rate": 0.049,
"bagging_freq": 5,
"bagging_fraction": 0.51,
"feature_fraction": 0.81,
"metric": "rmse"
}
model_lgb = lgb.train(params, train_set=dtrain, num_boost_round=999, valid_sets=watchlist, verbose_eval=101, early_stopping_rounds=21)
 models[site_id].append(model_lgb)
 y_pred_valid = model_lgb.predict(X_valid, num_iteration=model_lgb.best_iteration)
 y_pred_train_site[valid_index] = y_pred_valid
rmse = np.sqrt(mean_squared_error(y_valid, y_pred_valid))
print("Site Id:", site_id, ", Fold:", fold+1, ", RMSE:", rmse)
 score += rmse / cv
gc.collect()
cv_scores["site_id"].append(site_id)
 cv_scores["cv_score"].append(score)
 print("
Site Id:", site_id, ", CV RMSE:", np.sqrt(mean_squared_error(y_train_site, y_pred_train_site)) , "
" )<create_dataframe>
def keyword_disaster_probabilities(x):
tweets_w_keyword = np.sum(train["keyword"].fillna("" ).str.contains(x))
tweets_w_keyword_disaster = np.sum(train["keyword"].fillna("" ).str.contains(x)& train["target"] == 1)
 return tweets_w_keyword_disaster / tweets_w_keyword
keywords_vc["Disaster_Probability"] = keywords_vc.index.map(keyword_disaster_probabilities)
 keywords_vc.head()
pd.DataFrame.from_dict(cv_scores )<drop_column>
keywords_vc.sort_values(by="Disaster_Probability", ascending=False ).head(10 )
del df_train, X_train_site, y_train_site, X_train, y_train, dtrain, X_valid, y_valid, dvalid, y_pred_train_site, y_pred_valid, rmse, score, cv_scores
gc.collect()<drop_column>
len(train["location"].value_counts() )
weather_test = create_lag_features(weather_test, 18)
 weather_test.drop(["air_temperature", "cloud_coverage", "dew_temperature", "precip_depth_1_hr"], axis=1, inplace=True )<merge>
def create_corpus(target):
corpus = []
for w in train.loc[train["target"] == target]["text"].str.split() :
for i in w:
corpus.append(i)
 return corpus
def create_corpus_dict(target):
corpus = create_corpus(target)
 stop_dict = defaultdict(int)
 for word in corpus:
if word in stopwords:
stop_dict[word] += 1
return sorted(stop_dict.items() , key=lambda x:x[1], reverse=True )
df_test_sites = []
for site_id in tqdm(range(16), desc="site_id"):
print("Preparing test data for site_id", site_id)
 X_test_site = df_test[df_test.site_id==site_id]
weather_test_site = weather_test[weather_test.site_id==site_id]
X_test_site = X_test_site.merge(weather_test_site, on=["site_id", "timestamp"], how="left")
 row_ids_site = X_test_site.row_id
X_test_site = X_test_site[all_features]
y_pred_test_site = np.zeros(X_test_site.shape[0])
 print("Scoring for site_id", site_id)
 for fold in range(cv):
model_lgb = models[site_id][fold]
y_pred_test_site += model_lgb.predict(X_test_site, num_iteration=model_lgb.best_iteration)/ cv
gc.collect()
df_test_site = pd.DataFrame({"row_id": row_ids_site, "meter_reading": y_pred_test_site})
 df_test_sites.append(df_test_site)
 print("Scoring for site_id", site_id, "completed
")
 gc.collect()<feature_engineering>
corpus_disaster, corpus_non_disaster = create_corpus(1), create_corpus(0)
 counter_disaster, counter_non_disaster = Counter(corpus_disaster), Counter(corpus_non_disaster)
 x_disaster, y_disaster, x_non_disaster, y_non_disaster = [], [], [], []
counter = 0
for word, count in counter_disaster.most_common() [0:100]:
if(word not in stopwords and counter < 15):
counter += 1
x_disaster.append(word)
 y_disaster.append(count)
 counter = 0
for word, count in counter_non_disaster.most_common() [0:100]:
if(word not in stopwords and counter < 15):
counter += 1
x_non_disaster.append(word)
 y_non_disaster.append(count)
 fig, ax = plt.subplots(1,2, figsize=(20,8))
sns.barplot(x=y_disaster, y=x_disaster, orient='h', palette="Reds_d", ax=ax[0])
 sns.barplot(x=y_non_disaster, y=x_non_disaster, orient='h', palette="Blues_d", ax=ax[1])
 ax[0].set_title("Top 15 Non-Stopwords - Disaster Tweets")
 ax[0].set_xlabel("Word Frequency")
 ax[1].set_title("Top 15 Non-Stopwords - Non-Disaster Tweets")
 ax[1].set_xlabel("Word Frequency")
 plt.tight_layout()
plt.show()
df_test_site.loc[df_test_site['site_id']==0, 'meter_reading'] = df_test_site.loc[df_test_site['site_id']==0, 'meter_reading'] * 3.4118<save_to_csv>
def bigrams(target):
corpus = train[train["target"] == target]["text"]
count_vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
 bag_of_words = count_vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in count_vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq
submit = pd.concat(df_test_sites)
 submit.meter_reading = np.clip(np.expm1(submit.meter_reading), 0, a_max=None)
 submit.to_csv("submission_noleak.csv", index=False )<load_from_csv>
def remove_pattern(input_txt, pattern):
r = re.findall(pattern, input_txt)
 for i in r:
input_txt = re.sub(i, '', input_txt)
 return input_txt
leak0 = pd.read_csv("/kaggle/input/ashrae-leak-data/site0.csv")
 leak1 = pd.read_csv("/kaggle/input/ashrae-leak-data/site1.csv")
 leak2 = pd.read_csv("/kaggle/input/ashrae-leak-data/site2.csv")
 leak4 = pd.read_csv("/kaggle/input/ashrae-leak-data/site4.csv")
 leak15 = pd.read_csv("/kaggle/input/ashrae-leak-data/site15.csv")
 leak = pd.concat([leak0, leak1, leak2, leak4, leak15])
 del leak0, leak1, leak2, leak4, leak15, df_test_sites
gc.collect()
test = pd.read_csv(path_test)
 test = test[test.building_id.isin(leak.building_id.unique())]
leak = leak.merge(test, on=["building_id", "meter", "timestamp"])
 del test
gc.collect()
submit = submit.merge(leak[["row_id", "meter_reading_scraped"]], on=["row_id"], how="left")
 submit.loc[submit.meter_reading_scraped.notnull() , "meter_reading"] = submit.loc[submit.meter_reading_scraped.notnull() , "meter_reading_scraped"]
submit.drop(["meter_reading_scraped"], axis=1, inplace=True)
 submit.to_csv("submission.csv", index=False )<import_modules>
train['tweet'] = np.vectorize(remove_pattern )(train['text'], "
test['tweet'] = np.vectorize(remove_pattern )(test['text'], "
train.head()
import os, gc
import datetime
import numpy as np
import pandas as pd
import category_encoders
from sklearn.impute import SimpleImputer
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
from lightgbm import LGBMRegressor
from mlxtend.regressor import StackingRegressor
from pandas.api.types import is_categorical_dtype
from pandas.api.types import is_datetime64_any_dtype as is_datetime<define_variables>
train['tweet'] = train['tweet'].str.replace("[^a-zA-Z
test['tweet'] = test['tweet'].str.replace("[^a-zA-Z
train.head()
PATH = '.. /input/ashrae-energy-prediction/'<load_from_csv>
train['tweet'] = train['tweet'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))
test['tweet'] = test['tweet'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))
train.head()
train = pd.read_csv(f'{PATH}train.csv')
 weather_train = pd.read_csv(f'{PATH}weather_train.csv')
 metadata = pd.read_csv(f'{PATH}building_metadata.csv' )<filter>
train['tweet'] = train['tweet'].str.lower()
test['tweet'] = test['tweet'].str.lower()
train = train[train['building_id'] != 1099]
train = train.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )<define_variables>
set(stopwords.words('english'))
stops = set(stopwords.words('english'))
def weather_data_parser(weather_data)-> pd.DataFrame:
time_format = '%Y-%m-%d %H:%M:%S'
start_date = datetime.datetime.strptime(weather_data['timestamp'].min() , time_format)
 end_date = datetime.datetime.strptime(weather_data['timestamp'].max() , time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
for site_id in range(16):
site_hours = np.array(weather_data[weather_data['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list, site_hours), columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_data = pd.concat([weather_data, new_rows], sort=True)
 weather_data = weather_data.reset_index(drop=True)
 weather_data['datetime'] = pd.to_datetime(weather_data['timestamp'])
 weather_data['day'] = weather_data['datetime'].dt.day
weather_data['week'] = weather_data['datetime'].dt.week
weather_data['month'] = weather_data['datetime'].dt.month
weather_data = weather_data.set_index(['site_id', 'day', 'month'])
 air_temperature_filler = pd.DataFrame(weather_data.groupby(['site_id','day','month'])['air_temperature'].median() , columns=['air_temperature'])
 weather_data.update(air_temperature_filler, overwrite=False)
 cloud_coverage_filler = weather_data.groupby(['site_id', 'day', 'month'])['cloud_coverage'].median()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'), columns=['cloud_coverage'])
 weather_data.update(cloud_coverage_filler, overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_data.groupby(['site_id','day','month'])['dew_temperature'].median() , columns=['dew_temperature'])
 weather_data.update(due_temperature_filler, overwrite=False)
 sea_level_filler = weather_data.groupby(['site_id','day','month'])['sea_level_pressure'].median()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'), columns=['sea_level_pressure'])
 weather_data.update(sea_level_filler, overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_data.groupby(['site_id','day','month'])['wind_direction'].median() , columns=['wind_direction'])
 weather_data.update(wind_direction_filler, overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_data.groupby(['site_id','day','month'])['wind_speed'].median() , columns=['wind_speed'])
 weather_data.update(wind_speed_filler, overwrite=False)
 precip_depth_filler = weather_data.groupby(['site_id','day','month'])['precip_depth_1_hr'].median()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'), columns=['precip_depth_1_hr'])
 weather_data.update(precip_depth_filler, overwrite=False)
 weather_data = weather_data.reset_index()
weather_data = weather_data.drop(['datetime','day','week','month'], axis=1)
 return weather_data<string_transform>
train['tokenized_sents'] = train.apply(lambda row: nltk.word_tokenize(row['tweet']), axis=1)
 test['tokenized_sents'] = test.apply(lambda row: nltk.word_tokenize(row['tweet']), axis=1)
weather_train = weather_data_parser(weather_train )<categorify>
def remove_stops(row):
my_list = row['tokenized_sents']
meaningful_words = [w for w in my_list if not w in stops]
return(meaningful_words )
train = reduce_mem_usage(train, use_float16=True)
 weather_train = reduce_mem_usage(weather_train, use_float16=True)
 metadata = reduce_mem_usage(metadata, use_float16=True )<merge>
train['clean_tweet'] = train.apply(remove_stops, axis=1)
 test['clean_tweet'] = test.apply(remove_stops, axis=1)
 train.drop(["tweet","tokenized_sents"], axis = 1, inplace = True)
 test.drop(["tweet","tokenized_sents"], axis = 1, inplace = True)
train = train.merge(metadata, on='building_id', how='left')
 train = train.merge(weather_train, on=['site_id', 'timestamp'], how='left')
 del weather_train; gc.collect()<feature_engineering>
def rejoin_words(row):
my_list = row['clean_tweet']
joined_words =(" ".join(my_list))
return joined_words
train['clean_tweet'] = train.apply(rejoin_words, axis=1)
 test['clean_tweet'] = test.apply(rejoin_words, axis=1)
 train.head()
def data_parser(data)-> pd.DataFrame:
data.sort_values('timestamp')
 data.reset_index(drop=True)
 data['timestamp'] = pd.to_datetime(data['timestamp'], format='%Y-%m-%d %H:%M:%S')
 data['weekday'] = data['timestamp'].dt.weekday
data['hour'] = data['timestamp'].dt.hour
data['square_feet'] = np.log1p(data['square_feet'])
 data = data.drop(['timestamp', 'sea_level_pressure',
'wind_direction', 'wind_speed', 'year_built', 'floor_count'], axis=1)
 gc.collect()
encoder = LabelEncoder()
data['primary_use'] = encoder.fit_transform(data['primary_use'])
 return data<string_transform>
import gc
import time
import math
import random
import warnings
train = data_parser(train )<prepare_x_and_y>
warnings.filterwarnings("ignore" )
target = np.log1p(train['meter_reading'])
 features = train.drop(['meter_reading'], axis = 1)
 del train; gc.collect()<categorify>
import string
import folium
from colorama import Fore, Back, Style, init
categorical_features = ['building_id', 'site_id', 'meter', 'primary_use']
encoder = category_encoders.CountEncoder(cols=categorical_features)
 encoder.fit(features)
 features = encoder.transform(features)
 features_size = features.shape[0]
for feature in categorical_features:
features[feature] = features[feature] / features_size<train_model>
import scipy as sp
import networkx as nx
from pandas import Timestamp
from PIL import Image
from IPython.display import SVG
from keras.utils import model_to_dot
import requests
from IPython.display import HTML
imputer = SimpleImputer(missing_values=np.nan, strategy='median')
 imputer.fit(features)
 features = imputer.transform(features )<choose_model_class>
tqdm.pandas()
lightgbm = LGBMRegressor(objective='regression', learning_rate=0.05, num_leaves=1024,
feature_fraction=0.8, bagging_fraction=0.9, bagging_freq=5)
 ridge = Ridge(alpha=0.3)
 lasso = Lasso(alpha=0.3 )<find_best_model_class>
import plotly.express as px
import plotly.graph_objects as go
import plotly.figure_factory as ff
from plotly.subplots import make_subplots
import transformers
import tensorflow as tf
kfold = KFold(n_splits=2, shuffle=False)
 models = []
for idx,(train_idx, val_idx)in enumerate(kfold.split(features)) :
train_features, train_target = features[train_idx], target[train_idx]
val_features, val_target = features[val_idx], target[val_idx]
model = StackingRegressor(regressors=(lightgbm, ridge, lasso),
meta_regressor=lightgbm, use_features_in_secondary=True)
 model.fit(np.array(train_features), np.array(train_target))
models.append(model)
 print('RMSE: {:.4f} of fold: {}'.format(
 np.sqrt(mean_squared_error(val_target, model.predict(np.array(val_features)))) , idx))
del train_features, train_target, val_features, val_target; gc.collect()
del features, target; gc.collect()<load_from_csv>
from tensorflow.keras.callbacks import Callback
from sklearn.metrics import accuracy_score, roc_auc_score
from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger
test = pd.read_csv(f'{PATH}test.csv')
 weather_test = pd.read_csv(f'{PATH}weather_test.csv' )<drop_column>
from tensorflow.keras.models import Model
from kaggle_datasets import KaggleDatasets
from tensorflow.keras.optimizers import Adam
from tokenizers import BertWordPieceTokenizer
from tensorflow.keras.layers import Dense, Input, Dropout, Embedding
from tensorflow.keras.layers import LSTM, GRU, Conv1D, SpatialDropout1D
row_ids = test['row_id']
test.drop('row_id', axis=1, inplace=True )<feature_engineering>
from tensorflow.keras import layers
from tensorflow.keras import optimizers
from tensorflow.keras import activations
from tensorflow.keras import constraints
from tensorflow.keras import initializers
from tensorflow.keras import regularizers
import tensorflow.keras.backend as K
from tensorflow.keras.layers import *
from tensorflow.keras.optimizers import *
from tensorflow.keras.activations import *
from tensorflow.keras.constraints import *
from tensorflow.keras.initializers import *
from tensorflow.keras.regularizers import *
weather_test = weather_data_parser(weather_test )<normalization>
from sklearn import metrics
from sklearn.utils import shuffle
from gensim.models import Word2Vec
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer,HashingVectorizer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
from sklearn.tree import DecisionTreeClassifier
test = reduce_mem_usage(test, use_float16=True)
 weather_test = reduce_mem_usage(weather_test, use_float16=True )<merge>
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.tokenize import word_tokenize
from nltk.tokenize import TweetTokenizer
import nltk
from textblob import TextBlob
from nltk.corpus import wordnet
from nltk.corpus import stopwords
from nltk import WordNetLemmatizer
from nltk.stem import WordNetLemmatizer,PorterStemmer
from wordcloud import WordCloud, STOPWORDS
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk.tokenize import sent_tokenize, word_tokenize
test = test.merge(metadata, on='building_id', how='left')
 test = test.merge(weather_test, on=['site_id', 'timestamp'], how='left')
 del metadata; gc.collect()<categorify>
stopword=set(STOPWORDS)
 lem = WordNetLemmatizer()
tokenizer=TweetTokenizer()
np.random.seed(0)
 random_state = 42
test = data_parser(test)
 test = encoder.transform(test)
 for feature in categorical_features:
test[feature] = test[feature] / features_size
test = imputer.transform(test )<predict_on_test>
!pip install GPUtil
predictions = 0
for model in models:
predictions += np.expm1(model.predict(np.array(test)))/ len(models)
 del model; gc.collect()
del test, models; gc.collect()<save_to_csv>
from torch import nn
from transformers import AdamW, BertConfig, BertModel, BertTokenizer
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset, random_split
from transformers import get_linear_schedule_with_warmup
from sklearn.metrics import f1_score, accuracy_score
submission = pd.DataFrame({
'row_id': row_ids,
'meter_reading': np.clip(predictions, 0, a_max=None)
 })
 submission.to_csv('submission.csv', index=False, float_format='%.4f' )<import_modules>
def free_gpu_cache() :
print("Initial GPU Usage")
 gpu_usage()
torch.cuda.empty_cache()
cuda.select_device(0)
 cuda.close()
cuda.select_device(0)
 for obj in gc.get_objects() :
if torch.is_tensor(obj):
del obj
gc.collect()
print("GPU Usage after emptying the cache")
 gpu_usage()
import numpy as np
import sys
import pandas as pd
from skimage.io import imread, imsave
from skimage.color import gray2rgb
from skimage.color import rgb2gray
import matplotlib.pyplot as plt
from functools import reduce
import os<load_from_csv>
from torch import nn
from transformers import AdamW, BertConfig, BertModel, BertTokenizer
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset, random_split
from transformers import get_linear_schedule_with_warmup
from sklearn.metrics import f1_score, accuracy_score
sub = pd.read_csv(".. /input/ashrae-energy-prediction/sample_submission.csv" )<import_modules>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv" ).loc[:,["text","target"]]
train
<load_from_csv>
if torch.cuda.is_available() :
device = torch.device("cuda")
 else:
device = torch.device("cpu")
 device
df0 = pd.read_csv(".. /input/ashrae-public/ashrae-highway-kernel-route2.csv")
 df1 = pd.read_csv(".. /input/ashrae-public/ashrae-leak-validation-and-more.csv")
 df2 = pd.read_csv(".. /input/ashrae-public/ashrae-leak-validation-bruteforce-heuristic-search.csv")
 df3 = pd.read_csv(".. /input/ashrae-public/ashrae-may-make-it-up-to-1-0.csv")
 df4 = pd.read_csv(".. /input/ashrae-public/ashrae-maybe-this-can-make-public-lb-some-useful.csv")
 df5 = pd.read_csv(".. /input/ashrae-public/ashrae-simple-data-cleanup-lb-1-08-no-leaks.csv")
<define_variables>
dupli_sum = train.duplicated().sum()
if(dupli_sum>0):
print(dupli_sum, " duplicates found
removing...")
 train = train.loc[False==train.duplicated() , :]
else:
print("no duplicates found")
 train
blend1 = df0['meter_reading']*0.5 + df4['meter_reading']*0.3 + df5['meter_reading']*0.2<prepare_output>
X_train = train["text"].values
y_train = train["target"].values
sub['meter_reading'] = blend1
df11 = sub<define_variables>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
 lens = []
for text in X_train:
encoded_dict = tokenizer.encode_plus(text, add_special_tokens=True, return_tensors='pt')
 lens.append(encoded_dict['input_ids'].size() [1] )
blend2 = df11['meter_reading']*0.1 + df1['meter_reading']*0.15 + df2['meter_reading']*0.6 + df3['meter_reading']*0.15<load_from_csv>
sequence_length = 58
X_train_tokens = []
for text in X_train:
encoded_dict = tokenizer.encode_plus(text,
add_special_tokens=True,
max_length=sequence_length,
padding="max_length",
return_tensors='pt',
truncation=True)
 X_train_tokens.append(encoded_dict['input_ids'] )
sub1 = pd.read_csv(".. /input/ashrae-energy-prediction/sample_submission.csv" )<feature_engineering>
X_train_tokens = torch.cat(X_train_tokens, dim=0)
 y_train = torch.tensor(y_train )
sub1['meter_reading'] = blend2
sub1.head()<save_to_csv>
print('Original:
', X_train[5])
 print('Tokenization:
', X_train_tokens[5] )
sub1.to_csv(f'submission.csv', index=False, float_format='%g' )<import_modules>
batch_size = 32
dataset = TensorDataset(X_train_tokens, y_train.float())
 train_size = int(0.80 * len(dataset))
val_size = len(dataset)- train_size
train_set, val_set = random_split(dataset, [train_size, val_size])
 train_dataloader = DataLoader(train_set,
sampler=RandomSampler(train_set),
batch_size=batch_size)
 validation_dataloader = DataLoader(val_set,
sampler=RandomSampler(val_set),
batch_size=batch_size )
print(os.listdir(".. /input")) 
<load_from_csv>
bert = BertModel.from_pretrained("bert-base-uncased")
 bert.to(device)
 for batch in train_dataloader:
batch_features = batch[0].to(device)
 bert_output = bert(input_ids=batch_features)
 print("bert output: ", type(bert_output), len(bert_output))
print("first entry: ", type(bert_output[0]), bert_output[0].size())
 print("second entry: ", type(bert_output[1]), bert_output[1].size())
 break
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
train_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 MAX_SEQUENCE_LENGTH = 150
MAX_NB_WORDS = 100000
EMBEDDING_DIM = 300
VALIDATION_SPLIT = 0.1
num_lstm = 300
num_dense = 256
rate_drop_lstm = 0.2
rate_drop_dense = 0.2
act = 'relu'<set_options>
class BertClassifier(nn.Module):
def __init__(self):
super(BertClassifier, self ).__init__()
self.bert = BertModel.from_pretrained('bert-base-uncased')
 self.linear = nn.Linear(768, 1)
 self.sigmoid = nn.Sigmoid()
def forward(self, tokens):
bert_output = self.bert(input_ids=tokens)
 linear_output = self.linear(bert_output[1])
 proba = self.sigmoid(linear_output)
 return proba
color = sns.color_palette()
sns.set_style("dark" )<string_transform>
def eval(y_batch, probas):
preds_batch_np = np.round(probas.cpu().detach().numpy())
 y_batch_np = y_batch.cpu().detach().numpy()
acc = accuracy_score(y_true=y_batch_np, y_pred=preds_batch_np)
 f1 = f1_score(y_true=y_batch_np, y_pred=preds_batch_np, average='weighted')
 return acc, f1
def cleanData(text, stemming = False, lemmatize=False):
text = text.lower().split()
text = " ".join(text)
 text = re.sub(r"[^A-Za-z0-9^,!.\/'+\-=]", " ", text)
 text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r",", " ", text)
 text = re.sub(r"\.", " ", text)
 text = re.sub(r"!", " ! ", text)
 text = re.sub(r"\/", " ", text)
 text = re.sub(r"\^", " ^ ", text)
 text = re.sub(r"\+", " + ", text)
 text = re.sub(r"\-", " - ", text)
 text = re.sub(r"\=", " = ", text)
 text = re.sub(r"'", " ", text)
 text = re.sub(r"(\d+ )(k)", r"\g<1>000", text)
 text = re.sub(r":", " : ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r" u s ", " american ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e - mail", "email", text)
 text = re.sub(r"j k", "jk", text)
 text = re.sub(r"\s{2,}", " ", text)
 if stemming:
st = PorterStemmer()
txt = " ".join([st.stem(w)for w in text.split() ])
 if lemmatize:
wordnet_lemmatizer = WordNetLemmatizer()
txt = " ".join([wordnet_lemmatizer.lemmatize(w)for w in text.split() ])
 return text<feature_engineering>
def train(model, optimizer, scheduler, epochs, name):
history = []
best_f1 = 0
model.train()
for epoch in range(epochs):
print("=== Epoch: ", epoch+1, " / ", epochs, " ===")
 acc_total = 0
f1_total = 0
for it, batch in enumerate(train_dataloader):
x_batch, y_batch = [batch[0].to(device), batch[1].to(device)]
probas = torch.flatten(model(tokens=x_batch))
acc_f1_batch = eval(y_batch, probas)
 acc_total, f1_total = acc_total + acc_f1_batch[0], f1_total + acc_f1_batch[1]
model.zero_grad()
loss_func = nn.BCELoss()
batch_loss = loss_func(probas, y_batch)
 batch_loss.backward()
optimizer.step()
scheduler.step()
acc_total = acc_total/len(train_dataloader)
 f1_total = f1_total/len(train_dataloader)
 print("accuracy: ", acc_total, "
f1: ", f1_total)
 acc_val_total = 0
f1_val_total = 0
for batch in validation_dataloader:
x_batch, y_batch = [batch[0].to(device), batch[1].to(device)]
with torch.no_grad() :
probas = torch.flatten(model(tokens=x_batch))
acc_f1_val_batch = eval(y_batch, probas)
 acc_val_total, f1_val_total = acc_val_total + acc_f1_val_batch[0], f1_val_total + acc_f1_val_batch[1]
acc_val_total = acc_val_total/len(validation_dataloader)
 f1_val_total = f1_val_total/len(validation_dataloader)
 print("validation accuracy: ", acc_val_total, "
validation f1: ", f1_val_total, "
")
 if(f1_val_total>best_f1):
torch.save(model, name+".pt")
 best_f1 = f1_val_total
history.append({"acc":acc_total, "f1":f1_total, "acc_val":acc_val_total, "f1_val":f1_val_total})
 return [torch.load(name+".pt"), history]
print('Indexing word vectors')
 count = 0
embeddings_index = {}
f = open(EMBEDDING_FILE)
 for line in f:
values = line.split()
word = ' '.join(values[:-300])
 coefs = np.asarray(values[-300:], dtype='float32')
 embeddings_index[word] = coefs.reshape(-1)
 coef = embeddings_index[word]
f.close()
print('Found %d word vectors of glove.' % len(embeddings_index))
emb_mean,emb_std = coef.mean() , coef.std()
print(emb_mean,emb_std)
 print('Total %s word vectors.' % len(embeddings_index))<import_modules>
epochs = 10
baseline_bert_clf = BertClassifier()
baseline_bert_clf = baseline_bert_clf.to(device)
 adam = AdamW(baseline_bert_clf.parameters() , lr=5e-5, eps=1e-8)
 total_steps = len(train_dataloader)* epochs
sched = get_linear_schedule_with_warmup(adam, num_warmup_steps=0, num_training_steps=total_steps )
import os
import re
import csv
import codecs
import numpy as np
import pandas as pd
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from string import punctuation
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import *
from keras.layers import concatenate, CuDNNGRU
from keras.models import Model
from keras.layers.normalization import BatchNormalization
from keras.callbacks import EarlyStopping, ModelCheckpoint
import sys<feature_engineering>
baseline_bert_clf, history = train(model=baseline_bert_clf,
optimizer=adam,
scheduler=sched,
epochs=10,
name="baseline_bert_clf" )
print('Processing text dataset')
 train_df['comment_text'] = train_df['comment_text'].map(lambda x: cleanData(x, stemming=False, lemmatize=False))
test_df['comment_text'] = test_df['comment_text'].map(lambda x: cleanData(x, stemming=False, lemmatize=False))
special_character_removal=re.compile(r'[^a-z\d ]',re.IGNORECASE)
 replace_numbers=re.compile(r'\d+',re.IGNORECASE)
 def text_to_wordlist(text, remove_stopwords=False, stem_words=False):
text = text.lower().split()
if remove_stopwords:
stops = set(stopwords.words("english"))
text = [w for w in text if not w in stops]
text = " ".join(text)
 text=special_character_removal.sub('',text)
 text=replace_numbers.sub('n',text)
 if stem_words:
text = text.split()
stemmer = SnowballStemmer('english')
 stemmed_words = [stemmer.stem(word)for word in text]
text = " ".join(stemmed_words)
 return(text)
 list_sentences_train = train_df["comment_text"].fillna("NA" ).values
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train_df[list_classes].values
list_sentences_test = test_df["comment_text"].fillna("NA" ).values
comments = []
for text in list_sentences_train:
comments.append(text_to_wordlist(text))
test_comments=[]
for text in list_sentences_test:
test_comments.append(text_to_wordlist(text))
tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
 tokenizer.fit_on_texts(comments + test_comments)
 sequences = tokenizer.texts_to_sequences(comments)
 test_sequences = tokenizer.texts_to_sequences(test_comments)
 word_index = tokenizer.word_index
print('Found %s unique tokens' % len(word_index))
data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)
 print('Shape of data tensor:', data.shape)
 print('Shape of label tensor:', y.shape)
 test_data = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH)
 print('Shape of test_data tensor:', test_data.shape )<categorify>
history_df = pd.DataFrame(history)
 history_df
data_post = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH,padding='post', truncating='post')
 print('Shape of data tensor:', data_post.shape)
 print('Shape of label tensor:', y.shape)
 test_data_post = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')
 print('Shape of test_data tensor:', test_data_post.shape )<feature_engineering>
X_test = pd.read_csv(".. /input/nlp-getting-started/test.csv")["text"]
X_test_tokens = []
for text in X_test:
encoded_dict = tokenizer.encode_plus(text,
add_special_tokens=True,
max_length=sequence_length,
padding="max_length",
return_tensors='pt',
truncation=True)
 X_test_tokens.append(encoded_dict['input_ids'])
 X_test_tokens = torch.cat(X_test_tokens, dim=0)
 test_set = TensorDataset(X_test_tokens)
 test_dataloader = DataLoader(test_set,
sampler=SequentialSampler(test_set),
batch_size=batch_size )
print('Preparing embedding matrix')
 nb_words = min(MAX_NB_WORDS, len(word_index))
embedding_matrix = np.zeros(( nb_words, EMBEDDING_DIM))
for word, i in word_index.items() :
if i >= MAX_NB_WORDS:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print('Null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1)== 0))<define_variables>
X_test = pd.read_csv(".. /input/nlp-getting-started/test.csv")["text"]
X_test_tokens = []
for text in X_test:
encoded_dict = tokenizer.encode_plus(text,
add_special_tokens=True,
max_length=sequence_length,
padding="max_length",
return_tensors='pt',
truncation=True)
 X_test_tokens.append(encoded_dict['input_ids'])
 X_test_tokens = torch.cat(X_test_tokens, dim=0)
 test_set = TensorDataset(X_test_tokens)
 test_dataloader = DataLoader(test_set,
sampler=SequentialSampler(test_set),
batch_size=batch_size )
max_features=100000
maxlen=150
embed_size=300<compute_train_metric>
all_preds = []
for batch in test_dataloader:
x_batch = batch[0].to(device)
 with torch.no_grad() :
probas = baseline_bert_clf(tokens=x_batch)
 preds = np.round(probas.cpu().detach().numpy() ).astype(int ).flatten()
all_preds.extend(preds )
<init_hyperparams><EOS> 
challenge_pred = pd.concat([pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")["id"], pd.Series(all_preds)], axis=1)
 challenge_pred.columns = ['id', 'target']
challenge_pred.to_csv("submission.csv", index=False )