Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
tf.disable_eager_execution()
np.random.seed(10)
 warnings.filterwarnings("ignore")
 ps = PorterStemmer()
nlp=spacy.load("en_core_web_lg" )
best = pd.read_csv(".. /input/accuracy-subs/based_witch_time_by_store.csv")
 best.head()<load_from_csv>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
sales = pd.read_csv(".. /input/m5-forecasting-uncertainty/sales_train_evaluation.csv")
 sales.head()<merge>
train.isnull().sum()
sales["_all_"] = "Total"
sub = sales[["id", "item_id", "dept_id", "cat_id", "store_id", "state_id","_all_"]].merge(best, on='id')
 sub.shape<define_search_space>
train.isnull().sum()
qs = np.array([0.005,0.025,0.165,0.25, 0.5, 0.75, 0.835, 0.975, 0.995])
 def get_ratios(coef=0.15):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.norm.cdf(qs2)
 ratios /= ratios[4]
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 level_coef_dict = {
"_all_": get_ratios(coef=0.03),
"state_id": get_ratios(coef=0.04),
"store_id": get_ratios(coef=0.05),
"cat_id": get_ratios(coef=0.04),
"dept_id": get_ratios(coef=0.05), 
("state_id", "cat_id"): get_ratios(coef=0.05), 
("state_id", "dept_id"): get_ratios(coef=0.07), 
("store_id","cat_id"): get_ratios(coef=0.07), 
("store_id","dept_id"): get_ratios(coef=0.08),
"item_id": get_ratios(coef=0.11), 
("state_id", "item_id"): get_ratios(coef=0.15),
"id": get_ratios(coef=0.25)
 }<feature_engineering>
stopwords = list(STOP_WORDS)
 punct=string.punctuation
def text_data_cleaning(sentence):
doc = nlp(sentence)
 tokens = []
for token in doc:
if token.lemma_ != "-PRON-":
temp = token.lemma_.lower().strip()
else:
temp = token.lower_
tokens.append(temp)
 cleaned_tokens = []
for token in tokens:
if token not in stopwords and token not in punct:
cleaned_tokens.append(token)
 return " ".join(cleaned_tokens )
def quantile_coefs(q, level):
ratios = level_coef_dict[level]
return ratios.loc[q].values
def get_group_preds(pred, level):
df = pred.groupby(level)[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q, level)[:, None]
if level != "id":
df["id"] = [f"{lev}_X_{q:.3f}_evaluation" for lev, q in zip(df[level].values, q)]
else:
df["id"] = [f"{lev.replace('_evaluation', '')}_{q:.3f}_evaluation" for lev, q in zip(df[level].values, q)]
df = df[["id"]+list(cols)]
return df
def get_couple_group_preds(pred, level1, level2):
df = pred.groupby([level1, level2])[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q,(level1, level2)) [:, None]
df["id"] = [f"{lev1}_{lev2}_{q:.3f}_evaluation" for lev1,lev2, q in
zip(df[level1].values,df[level2].values, q)]
df = df[["id"]+list(cols)]
return df
levels = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id", "_all_"]
couples = [("state_id", "item_id"),("state_id", "dept_id"),("store_id","dept_id"), 
("state_id", "cat_id"),("store_id","cat_id")]
cols = [f"F{i}" for i in range(1, 29)]
df = []
for level in levels :
df.append(get_group_preds(sub, level))
for level1,level2 in couples:
df.append(get_couple_group_preds(sub, level1, level2))
df = pd.concat(df, axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df = pd.concat([df,df] , axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df.loc[df.index >= len(df.index)//2, "id"] = df.loc[df.index >= len(df.index)//2, "id"].str.replace(
 "_evaluation$", "_validation")
 df.to_csv("submission.csv", index = False )<import_modules>

 train['text'] = train.text.apply(lambda x: text_data_cleaning(x))
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
from tqdm import tqdm
import gc<categorify>
train.keyword = train.keyword.fillna("")
 train['new_text'] = train.text
test.keyword = test.keyword.fillna("")
 test['text'] = test.text
test['text'] = test.text.apply(lambda x: text_data_cleaning(x))
def preprocess_sales(sales, start=1400, upper=1970):
if start is not None:
print("dropping...")
 to_drop = [f"d_{i+1}" for i in range(start-1)]
print(sales.shape)
 sales.drop(to_drop, axis=1, inplace=True)
 print(sales.shape)
 print("adding...")
 new_columns = ['d_%i'%i for i in range(1942, upper, 1)]
for col in new_columns:
sales[col] = np.nan
print("melting...")
 sales = sales.melt(id_vars=["id", "item_id", "dept_id", "cat_id", "store_id", "state_id","scale","start"],
var_name='d', value_name='demand')
 print("generating order")
 if start is not None:
skip = start
else:
skip = 1
sales["nb"] =sales.index // 42840 + skip
return sales
def preprocess_calendar(calendar):
global maps, mods
calendar["event_name"] = calendar["event_name_1"]
calendar["event_type"] = calendar["event_type_1"]
map1 = {mod:i for i,mod in enumerate(calendar['event_name'].unique())}
calendar['event_name'] = calendar['event_name'].map(map1)
 map2 = {mod:i for i,mod in enumerate(calendar['event_type'].unique())}
calendar['event_type'] = calendar['event_type'].map(map2)
 calendar['nday'] = calendar['date'].str[-2:].astype(int)
 maps["event_name"] = map1
maps["event_type"] = map2
mods["event_name"] = len(map1)
 mods["event_type"] = len(map2)
 calendar["wday"] -=1
calendar["month"] -=1
calendar["year"] -= 2011
mods["month"] = 12
mods["year"] = 6
mods["wday"] = 7
mods['snap_CA'] = 2
mods['snap_TX'] = 2
mods['snap_WI'] = 2
calendar.drop(["event_name_1", "event_name_2", "event_type_1", "event_type_2", "date", "weekday"],
axis=1, inplace=True)
 return calendar
def make_dataset(categorize=False ,start=1400, upper= 1970):
global maps, mods
print("loading calendar...")
 calendar = pd.read_csv(".. /input/m5-forecasting-uncertainty/calendar.csv")
 print("loading sales...")
 sales = pd.read_csv(".. /input/walmartadd/sales.csv")
 cols = ["item_id", "dept_id", "cat_id","store_id","state_id"]
if categorize:
for col in cols:
temp_dct = {mod:i for i, mod in enumerate(sales[col].unique())}
mods[col] = len(temp_dct)
 maps[col] = temp_dct
for col in cols:
sales[col] = sales[col].map(maps[col])
 sales =preprocess_sales(sales, start=start, upper= upper)
 calendar = preprocess_calendar(calendar)
 calendar = reduce_mem_usage(calendar)
 print("merge with calendar...")
 sales = sales.merge(calendar, on='d', how='left')
 del calendar
print("reordering...")
 sales.sort_values(by=["id","nb"], inplace=True)
 print("re-indexing.. ")
 sales.reset_index(inplace=True, drop=True)
 gc.collect()
sales['n_week'] =(sales['nb']-1)//7
sales["nday"] -= 1
mods['nday'] = 31
sales = reduce_mem_usage(sales)
 gc.collect()
return sales 
<create_dataframe>
X_train, X_test, y_train, y_test = train_test_split(train.new_text.values, train.target.values, random_state = 42, test_size=0.2)
 test_data = test.text.values
%%time
CATEGORIZE = True;
START = 1400; UPPER = 1970;
maps = {}
mods = {}
sales = make_dataset(categorize=CATEGORIZE ,start=START, upper= UPPER )<feature_engineering>
module_url = "https://tfhub.dev/google/universal-sentence-encoder/4"
embed = hub.KerasLayer(module_url, trainable=False, name='USE_embedding' )
sales["x"] = sales["demand"] / sales["scale"]<define_variables>
def build_model(embed):
model = Sequential([
Input(shape=[], dtype=tf.string),
embed,
Dense(1024, activation='elu'),
BatchNormalization() ,
Dropout(0.5),
Dense(512, activation='elu'),
BatchNormalization() ,
Dropout(0.35),
Dense(256, activation='relu'),
BatchNormalization() ,
Dropout(0.1),
Dense(1, activation='sigmoid')
 ])
 model.compile(Adam(lr=0.001), loss='binary_crossentropy', metrics=['accuracy'])
 return model
nb = sales['nb'].values
MAX_LAG = max(LAGS)
 tr_mask = np.logical_and(nb>START + MAX_LAG, nb<=1941)
 val_mask = np.logical_and(nb>1913, nb<=1941)
 te_mask = np.logical_and(nb>1941, nb<=1969 )<prepare_x_and_y>
earlyStopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1, mode='min')
 mcp_save = ModelCheckpoint('model.hdf5', save_best_only=True, monitor='val_loss', mode='min')
 reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=7, verbose=2, epsilon=1e-4, mode='min' )
scale = sales['scale'].values
ids = sales['id'].values
ys = sales['x'].values
Z = sales[FEATS].values<train_model>
with tf.compat.v1.Session() as session:
tf.compat.v1.keras.backend.set_session(session)
 session.run([tf.compat.v1.global_variables_initializer() , tf.compat.v1.tables_initializer() ])
 history = model.fit(
 X_train, y_train,
validation_data=(X_test,y_test),
epochs=35,
callbacks=[earlyStopping,reduce_lr_loss,mcp_save],
batch_size=128
)
scale<normalization>
with tf.Session() as session:
tf.compat.v1.keras.backend.set_session(session)
 session.run(tf.global_variables_initializer())
 session.run(tf.tables_initializer())
 model.load_weights('model.hdf5')
 y_pred = model.predict(X_test)
 print(classification_report(y_test, y_pred.round().astype(int)) )
<prepare_x_and_y><EOS> 
with tf.Session() as session:
tf.compat.v1.keras.backend.set_session(session)
 session.run(tf.global_variables_initializer())
 session.run(tf.tables_initializer())
 model.load_weights('model.hdf5')
 sub = model.predict(test_data)
 subm = pd.DataFrame()
subm['id'] = test['id']
subm['target'] = sub.round().astype(int)
 subm.to_csv("pred.csv", index = False )