Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<train_model>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
print('Total training time: ', datetime.now() - start_time)
 i=0
for mol_type in mol_types:
print(mol_type,": cv score is ",cv_score[i])
 i+=1
print("total cv score is",cv_score_total )<save_to_csv>
import os
import sys
import logging
import itertools
import re
import pandas as pd
import numpy as np
import sklearn.metrics
import sklearn.preprocessing
import nltk
import tensorflow as tf
import tensorflow_hub as hub
import tokenization
import matplotlib.pyplot as plt
import plotly
import plotly.graph_objects as go
import plotly.offline
import wordcloud
def submits(predictions):
submit["scalar_coupling_constant"] = predictions
submit.to_csv("/kaggle/working/submission.csv", index=False)
 submits(test_prediction )<set_options>
plotly.offline.init_notebook_mode(connected=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_rows = 500
pd.options.display.max_columns = None
pd.options.display.max_colwidth = 160
%matplotlib inline 
<define_variables>
log = logging.getLogger(name=__name__)
 log.setLevel(logging.INFO)
 logging.captureWarnings(True)
 formatter = logging.Formatter(
 '%(asctime)s - %(name)s - %(levelname)s - %(message)s' 
)
 stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
 stream_handler.setFormatter(formatter)
 log.addHandler(stream_handler)
 log.info(f"Python version: {sys.version}")
 log.info(f"Numpy version: {np.__version__}")
 log.info(f"Pandas version: {pd.__version__}")
 log.info(f"Scikit-learn version: {sklearn.__version__}")
 log.info(f"TensorFlow version: {tf.__version__}")
 log.info(f"Plotly version: {plotly.__version__}")
 log.info(f"WordCloud version: {wordcloud.__version__}" )
DATA_PATH = '.. /input'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}<set_options>
SEED = 1
tf.random.set_seed(SEED)
 log.info(f"tensorflow.random seed: {SEED}" )
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 120)
 pd.set_option('display.max_columns', 120 )<load_from_csv>
SUCCESS = 0
UNK = "UNK"
NUM = "number"
AT = "recipient"
http = "http"
html = "html"
target = "target"
keyword = "keyword"
old_text = "text"
location = "location"
text = "t"
hashtag = "hashtag"
at = "at"
href = "href"
y_cols = [f"{target}_0", f"{target}_1"]
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 train_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]
train_csv.head(10 )<load_from_csv>
try:
nltk.download('stopwords')
 except:
log.error('...')
 try:
stopwords =(nltk.corpus.stopwords.words("english")
 + ["u", "im", "st", "nd", "rd", "th"] 
)
 except:
log.error('...' )
submission_csv = pd.read_csv(f'{DATA_PATH}/sample_submission.csv', index_col='id' )<load_from_csv>
data_dir = ".. /input/nlp-getting-started"
log.info(f"Data directory: {data_dir}")
 train_bn = "train.csv"
test_bn = "test.csv"
train_fn = os.path.join(data_dir, train_bn)
 test_fn = os.path.join(data_dir, test_bn)
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 test_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]
test_csv.head(10 )<data_type_conversions>
df_train = pd.read_csv(train_fn)
 df_test = pd.read_csv(test_fn)
 log.info(f"Training data shape: {df_train.shape}")
 log.info(f"Test data shape: {df_test.shape}")
 train_pts = df_train.shape[0]
structures_dtypes = {
'molecule_name': 'category',
'atom_index': 'int8',
'atom': 'category',
'x': 'float32',
'y': 'float32',
'z': 'float32'
}
structures_csv = pd.read_csv(f'{DATA_PATH}/structures.csv', dtype=structures_dtypes)
 structures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 structures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]
structures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS ).astype('int8')
 structures_csv.head(10 )<data_type_conversions>
def to_lower(df, col=text):

 df[col] = df[col].apply(lambda x: x.casefold())
 return SUCCESS
def build_type_dataframes(base, structures, coupling_type):
base = base[base['type'] == coupling_type].drop('type', axis=1 ).copy()
base = base.reset_index()
base['id'] = base['id'].astype('int32')
 structures = structures[structures['molecule_index'].isin(base['molecule_index'])]
return base, structures<merge>
def hash_handling(df, col=text):

 reg_hash_full = re.compile("(
 reg_hash = re.compile("(
 f = lambda x: [y.group() for y in reg_hash_full.finditer(x)]
g = lambda x: ' '.join(x)
 df[hashtag] = df[col].apply(f ).apply(g)
 df[col] = df[col].apply(lambda x: reg_hash.sub(' ', x))
return SUCCESS
def add_coordinates(base, structures, index):
df = pd.merge(base, structures, how='inner',
left_on=['molecule_index', f'atom_index_{index}'],
right_on=['molecule_index', 'atom_index'] ).drop(['atom_index'], axis=1)
 df = df.rename(columns={
'atom': f'atom_{index}',
'x': f'x_{index}',
'y': f'y_{index}',
'z': f'z_{index}'
})
 return df<merge>
def at_handling(df, col=text, at_col=at):

 reg_at = re.compile("(@)")
 reg_at_full = re.compile("(@)\w+")
 f = lambda x: [y.group() for y in reg_at_full.finditer(x)]
g = lambda x: ' '.join(x)
 df[at_col] = df[col].apply(f ).apply(g)
 df[col] = df[col].apply(lambda x: reg_at_full.sub(f" {AT} ", x))
return SUCCESS
def add_atoms(base, atoms):
df = pd.merge(base, atoms, how='inner',
on=['molecule_index', 'atom_index_0', 'atom_index_1'])
 return df<merge>
def href_handling(df, col=text, new_col=href):

 reg_href_full = re.compile("(htt)\S+")
 f = lambda x: len(list(reg_href_full.finditer(x)))
 df[new_col] = df[col].apply(f)
 df[col] = df[col].apply(lambda x: reg_href_full.sub(f' {http} ', x))
return SUCCESS
def merge_all_atoms(base, structures):
df = pd.merge(base, structures, how='left',
left_on=['molecule_index'],
right_on=['molecule_index'])
 df = df[(df.atom_index_0 != df.atom_index)&(df.atom_index_1 != df.atom_index)]
return df<feature_engineering>
def html_special_handling(df, col=text):

 reg_html = re.compile("(&)\w+(;)")
 df[col] = df[col].apply(lambda x: reg_html.sub(f' {html} ', x))
return SUCCESS
def add_center(df):
df['x_c'] =(( df['x_1'] + df['x_0'])* np.float32(0.5))
df['y_c'] =(( df['y_1'] + df['y_0'])* np.float32(0.5))
df['z_c'] =(( df['z_1'] + df['z_0'])* np.float32(0.5))
def add_distance_to_center(df):
df['d_c'] =(( 
(df['x_c'] - df['x'])**np.float32(2)+ 
(df['y_c'] - df['y'])**np.float32(2)+ 
(df['z_c'] - df['z'])**np.float32(2)
)**np.float32(0.5))
def add_distance_between(df, suffix1, suffix2):
df[f'd_{suffix1}_{suffix2}'] =(( 
(df[f'x_{suffix1}'] - df[f'x_{suffix2}'])**np.float32(2)+ 
(df[f'y_{suffix1}'] - df[f'y_{suffix2}'])**np.float32(2)+ 
(df[f'z_{suffix1}'] - df[f'z_{suffix2}'])**np.float32(2)
)**np.float32(0.5))<compute_test_metric>
def xc2x89_byte_handling(df, col=text):

 reg_x89 = re.compile(b"\xc2\x89".decode('utf-8')+"\S+")
 df[col] = df[col].apply(lambda x: reg_x89.sub(' ', x))
return SUCCESS
def add_distances(df):
n_atoms = 1 + max([int(c.split('_')[1])for c in df.columns if c.startswith('x_')])
 for i in range(1, n_atoms):
for vi in range(min(4, i)) :
add_distance_between(df, i, vi )<merge>
def special_char_handling(df, col=text):

 reg_special = re.compile("[^\w\s]")
 df[col] = df[col].apply(lambda x: reg_special.sub(' ', x))
df[col] = df[col].apply(lambda x: re.sub('_', ' ', x))
return SUCCESS
def add_n_atoms(base, structures):
dfs = structures['molecule_index'].value_counts().rename('n_atoms' ).to_frame()
return pd.merge(base, dfs, left_on='molecule_index', right_index=True )<drop_column>
def contraction_handling(df, col=text):

 reg_contract = re.compile("\s(s|m|t|(nt)|(ve)|w)\s")
 df[col] = df[col].apply(lambda x: reg_contract.sub(' ', x))
return SUCCESS
def build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=10):
base, structures = build_type_dataframes(some_csv, structures_csv, coupling_type)
 base = add_coordinates(base, structures, 0)
 base = add_coordinates(base, structures, 1)
 base = base.drop(['atom_0', 'atom_1'], axis=1)
 atoms = base.drop('id', axis=1 ).copy()
if 'scalar_coupling_constant' in some_csv:
atoms = atoms.drop(['scalar_coupling_constant'], axis=1)
 add_center(atoms)
 atoms = atoms.drop(['x_0', 'y_0', 'z_0', 'x_1', 'y_1', 'z_1'], axis=1)
 atoms = merge_all_atoms(atoms, structures)
 add_distance_to_center(atoms)
 atoms = atoms.drop(['x_c', 'y_c', 'z_c', 'atom_index'], axis=1)
 atoms.sort_values(['molecule_index', 'atom_index_0', 'atom_index_1', 'd_c'], inplace=True)
 atom_groups = atoms.groupby(['molecule_index', 'atom_index_0', 'atom_index_1'])
 atoms['num'] = atom_groups.cumcount() + 2
atoms = atoms.drop(['d_c'], axis=1)
 atoms = atoms[atoms['num'] < n_atoms]
atoms = atoms.set_index(['molecule_index', 'atom_index_0', 'atom_index_1', 'num'] ).unstack()
atoms.columns = [f'{col[0]}_{col[1]}' for col in atoms.columns]
atoms = atoms.reset_index()
for col in atoms.columns:
if col.startswith('atom_'):
atoms[col] = atoms[col].fillna(0 ).astype('int8')
 atoms['molecule_index'] = atoms['molecule_index'].astype('int32')
 full = add_atoms(base, atoms)
 add_distances(full)
 full.sort_values('id', inplace=True)
 return full<define_variables>
def encode_numerals(df, col=text):

 reg_numerals = re.compile("\d+[\s\d]*")
 df[col] = df[col].apply(lambda x: reg_numerals.sub(f' {NUM} ', x))
return SUCCESS
def take_n_atoms(df, n_atoms, four_start=4):
labels = []
for i in range(2, n_atoms):
label = f'atom_{i}'
labels.append(label)
 for i in range(n_atoms):
num = min(i, 4)if i < four_start else 4
for j in range(num):
labels.append(f'd_{i}_{j}')
 if 'scalar_coupling_constant' in df:
labels.append('scalar_coupling_constant')
 return df[labels]<split>
def remove_stopwords(df, col=text, to_remove=stopwords):

 f =(lambda x:
' '.join([y for y in x.strip().split() if y not in to_remove])
)
 df[col] = df[col].apply(f)
 return SUCCESS
%%time
def type_select(types = '1JHN'):
full = build_couple_dataframe(train_csv, structures_csv, types, n_atoms=10)
 print(full.shape)
 df = take_n_atoms(full, 7)
 df = df.fillna(0)
 X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 X_train, X_val, y_train, y_val = train_test_split(X_data, y_data, test_size=0.2, random_state=128)
 print(X_train.shape, X_val.shape, y_train.shape, y_val.shape)
 return X_train, X_val, y_train, y_val 
<find_best_params>
def preprocess(df, col=text, old_col=old_text):

 df[col] = df[old_col]
to_lower(df)
 hash_handling(df)
 at_handling(df)
 href_handling(df)
 html_special_handling(df)
 xc2x89_byte_handling(df)
 special_char_handling(df)
 contraction_handling(df)
 remove_stopwords(df)
 encode_numerals(df)
 return SUCCESS
model_params = {
'1JHN': 7,
'1JHC': 10,
'2JHH': 9,
'2JHN': 9,
'2JHC': 9,
'3JHH': 9,
'3JHC': 10,
'3JHN': 10
}
model_params.keys()<prepare_x_and_y>
log.info(f"Training set preprocessing status: {preprocess(df_train)}.")
 log.info(f"Test set preprocessing status: {preprocess(df_test)}." )
X_train, X_val, y_train, y_val = type_select(types = '3JHN' )<init_hyperparams>
ave_words_positive =(
 df_train.loc[df_train[target]==1, text].apply(lambda x: len(x.split())) 
.sum()
/ df_train.loc[df_train[target]==1, text].count() 
)
 ave_words_negative =(
 df_train.loc[df_train[target]==0, text].apply(lambda x: len(x.split())) 
.sum()
/ df_train.loc[df_train[target]==0, text].count() 
)
 log.info(f"Average words in a positive example {ave_words_positive}.
 "
f"Average words in a negative example {ave_words_negative}."
)
%%time
LGB_PARAMS = {
'objective': 'regression',
'metric': 'mae',
'verbosity': -1,
'boosting_type': 'gbdt',
'learning_rate': 0.1455,
'num_leaves': 129,
'min_child_samples': 78,
'max_depth': 13,
'subsample_freq': 1,
'subsample': 0.88,
'bagging_seed': 15,
'reg_alpha': 0.10107001,
'reg_lambda': 0.300132,
'colsample_bytree': 1.0
}
LGB_PARAMS={'bagging_seed': 14, 'colsample_bytree': 1.0, 'learning_rate': 0.14848931924611134, 'max_depth': -1, 'min_child_samples': 80, 'categorical_feature':categorical_feature, 'num_leaves': 129, 'random_state': 42, 'reg_alpha': 0.1, 'reg_lambda': 0.3, 'subsample': 0.89, 'subsample_freq': 1, 'verbosity': -1}
model = LGBMRegressor(**LGB_PARAMS, n_estimators=3000, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_val, y_val)], eval_metric='mae',
verbose=400, early_stopping_rounds=500)
 y_pred = model.predict(X_val)
 np.log(mean_absolute_error(y_val, y_pred))<define_search_space>
tokenize_flatten = lambda series:(
 list(itertools.chain(*[x.split() for x in series])) 
)
 wc_size =(14, 14)
 tdf = df_train[df_train[target]==1]
unique_words, word_counts =(
 np.unique(tokenize_flatten(tdf[text]), return_counts=True)
)
 sm = np.sum(word_counts)
 frequency_dict = {
x: word_counts[i]/sm
for i, x in np.ndenumerate(unique_words)
 }
try:
frequency_dict.pop(NUM)
 except:
log.error(NUM)
 try:
frequency_dict.pop(AT)
 except:
log.error(AT)
 try:
frequency_dict.pop(http)
 except:
log.error(http)
 try:
frequency_dict.pop(html)
 except:
log.error(html)
 try:
frequency_dict.pop(UNK)
 except:
log.error(UNK)
 wc =(
 wordcloud.WordCloud(
 width=1000, height=1000,
background_color='white',
min_font_size=10
).generate_from_frequencies(frequency_dict)
)
 fig = plt.figure(figsize=wc_size, facecolor=None)
 ax = fig.add_subplot()
ax.set_title("Words in the disaster tweets")
 _ = ax.imshow(wc )
categorical_feature=[0,1,2,3,4]<init_hyperparams>
def bigrams_count(df, col, top_n=10):

 words = [x.split() for x in df[col]]
bigrams = [x[i]+"_"+x[i+1] for x in words for i in range(len(x)-1)]
uniq_pairs, counts = np.unique(np.array(bigrams), return_counts=True)
 return np.array([uniq_pairs, counts] )
LGB_PARAMS_3JHN={'bagging_seed': 14, 'colsample_bytree': 1.0, 'learning_rate': 0.14548931924611134, 'max_depth': 14, 'min_child_samples': 80, 'num_leaves': 129, 'random_state': 42, 'reg_alpha': 0.1, 'reg_lambda': 0.3, 'subsample': 0.89, 'subsample_freq': 1, 'verbosity': -1}<categorify>
df_bi_0 = pd.DataFrame(
 bigrams_count(df_train[df_train[target]==0], text ).T,
columns=["bigram", "count"] 
)
 df_bi_1 = pd.DataFrame(
 bigrams_count(df_test[df_train[target]==0], text ).T,
columns=["bigram", "count"] 
)
 df_bi = df_bi_0.merge(df_bi_1, how="outer", left_on="bigram",
suffixes=("_0", "_1"), right_on="bigram")
 df_bi.fillna(0, inplace=True)
 df_bi = df_bi.astype({"count_0": int, "count_1": int})
 df_bi["total"] = df_bi["count_0"] + df_bi["count_1"]
<prepare_x_and_y>
top_n = 100
top_bigrams = df_bi.nlargest(top_n, "total" )
def build_x_y_data(some_csv, coupling_type, n_atoms):
full = build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=n_atoms)
 df = take_n_atoms(full, n_atoms)
 df = df.fillna(0)
 print(df.columns)
 if 'scalar_coupling_constant' in df:
X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 else:
X_data = df.values.astype('float32')
 y_data = None
return X_data, y_data<find_best_model_class>
def tokenize_dataframe(df, col, max_len=20):

 df_tmp = pd.DataFrame(
 df[col].apply(lambda x: reversed(x.split())).tolist() 
)
 orig_len = len(df_tmp.columns)
 df_tmp = df_tmp.rename(
 lambda x: f'{col}_{max_len-1-x:02d}',
axis=1 
)
 enum_cols = [f'{col}_{i:02d}' for i in range(max_len)]
if orig_len < max_len:
compl_cols = [x for x in enum_cols if x not in df_tmp.columns]
df_tmp[compl_cols] = np.nan
df[enum_cols] = df_tmp[enum_cols]
return enum_cols
def train_and_predict_for_one_coupling_type(coupling_type, submission, n_atoms, n_folds=4, n_splits=4, random_state=128):
print(f'*** Training Model for {coupling_type} ***')
 X_data, y_data = build_x_y_data(train_csv, coupling_type, n_atoms)
 X_test, _ = build_x_y_data(test_csv, coupling_type, n_atoms)
 y_pred = np.zeros(X_test.shape[0], dtype='float32')
 cv_score = 0
if n_folds > n_splits:
n_splits = n_folds
kfold = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)
 print('fitting {} with {} LGB_PARAMS'.format(coupling_type,LGB_PARAMS))
for fold,(train_index, val_index)in enumerate(kfold.split(X_data, y_data)) :
if fold >= n_folds:
break
X_train, X_val = X_data[train_index], X_data[val_index]
y_train, y_val = y_data[train_index], y_data[val_index]
model = LGBMRegressor(**LGB_PARAMS, n_estimators= 9000, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_val, y_val)], eval_metric='mae',
verbose=1000, early_stopping_rounds=500)
 y_val_pred = model.predict(X_val)
 val_score = np.log(mean_absolute_error(y_val, y_val_pred))
print(f'{coupling_type} Fold {fold}, logMAE: {val_score}')
 cv_score += val_score / n_folds
y_pred += model.predict(X_test)/ n_folds
plt.scatter(y_val, y_val_pred)
 plt.show()
submission.loc[test_csv['type'] == coupling_type, 'scalar_coupling_constant'] = y_pred
return cv_score<init_hyperparams>
def transform_data(df, col=text):

 word_cols = tokenize_dataframe(df, col, max_len=25)
 df[word_cols] = df[word_cols].fillna('')
 lemmatizer = nltk.stem.WordNetLemmatizer()
ps = nltk.stem.PorterStemmer()
df[word_cols] = df[word_cols].applymap(lambda x: ps.stem(x))
df[word_cols] = df[word_cols].applymap(lambda x: lemmatizer.lemmatize(x))
_ = filter_infrequent(df, word_cols, cutoff=10)
 word_enc = sklearn.preprocessing.LabelEncoder()
df[word_cols] =(word_enc 
.fit_transform(df[word_cols].values.flatten())
.reshape(df[word_cols].shape)
)
 num_unique_words = word_enc.classes_.shape[0]
return word_enc, num_unique_words, word_cols
model_params = {
'1JHN': 7,
'1JHC': 10,
'2JHH': 9,
'2JHN': 9,
'2JHC': 9,
'3JHH': 9,
'3JHC': 10,
'3JHN': 10
}
cat_code = {'1JHN': 5,'1JHC': 7,'2JHH': 7,
'2JHN': 7,'2JHC': 7,'3JHH': 7,
'3JHC': 8,'3JHN': 7}
categorical_feature = [0,1,2,3,4,5,6]
LGB_PARAMS_2JHC={'bagging_seed': 14, 'colsample_bytree': 1.0, 'learning_rate': 0.14548931824400012, 'max_depth': -1, 'min_child_samples': 81,'categorical_feature':categorical_feature, 'num_leaves': 129, 'random_state': 42, 'reg_alpha': 0.1, 'reg_lambda': 0.30, 'subsample': 0.89, 'subsample_freq': 1, 'verbosity': -1}
LGB_PARAMS_3JHN={'bagging_seed': 14, 'colsample_bytree': 1.0, 'learning_rate': 0.1454893092, 'max_depth': -1, 'min_child_samples': 81, 'num_leaves': 129, 'random_state': 42, 'reg_alpha': 0.1, 'reg_lambda': 0.3, 'subsample': 0.89, 'subsample_freq': 1, 'verbosity': -1}
LGB_PARAMS_2JHN={'bagging_seed': 14, 'colsample_bytree': 1.0, 'learning_rate': 0.1454893072, 'max_depth': -1, 'min_child_samples': 82, 'num_leaves': 130, 'random_state': 42, 'reg_alpha': 0.1, 'reg_lambda': 0.3, 'subsample': 0.89, 'subsample_freq': 1, 'verbosity': -1}
N_FOLDS = 5
submission = submission_csv.copy()
cv_scores = {}
for coupling_type in model_params.keys() :
if(coupling_type == '2JHC'):
categorical_feature = [0,1,2,3,4,5,6]
LGB_PARAMS = {'bagging_seed': 14, 'colsample_bytree': 1.0, 'learning_rate': 0.14548931824400012, 'max_depth': -1, 'min_child_samples': 81,'categorical_feature':categorical_feature, 'num_leaves': 129, 'random_state': 42, 'reg_alpha': 0.1, 'reg_lambda': 0.30, 'subsample': 0.89, 'subsample_freq': 1, 'verbosity': -1}
elif(coupling_type == '3JHN'):
categorical_feature = [0,1,2,3,4,5,6]
LGB_PARAMS = {'bagging_seed': 14, 'colsample_bytree': 1.0, 'learning_rate': 0.1454893092, 'max_depth': -1, 'min_child_samples': 81, 'num_leaves': 129, 'random_state': 42, 'reg_alpha': 0.1, 'reg_lambda': 0.3, 'subsample': 0.89, 'subsample_freq': 1, 'verbosity': -1}
elif(coupling_type == '2JHN'):
categorical_feature = [0,1,2,3,4,5,6]
LGB_PARAMS = {'bagging_seed': 14, 'colsample_bytree': 1.0, 'learning_rate': 0.1454893072, 'max_depth': -1, 'min_child_samples': 82, 'num_leaves': 130, 'random_state': 42, 'reg_alpha': 0.1, 'reg_lambda': 0.3, 'subsample': 0.89, 'subsample_freq': 1, 'verbosity': -1}
elif(coupling_type == '3JHC'):
categorical_feature = [0,1,2,3,4,5,6,7]
LGB_PARAMS={'bagging_seed': 14, 'colsample_bytree': 1.0, 'learning_rate': 0.14548931824400012, 'max_depth': -1, 'min_child_samples': 81,'categorical_feature':categorical_feature, 'num_leaves': 129, 'random_state': 42, 'reg_alpha': 0.1, 'reg_lambda': 0.30, 'subsample': 0.89, 'subsample_freq': 1, 'verbosity': -1}
elif(coupling_type == '1JHN'):
categorical_feature = [0,1,2,3,4]
LGB_PARAMS={'bagging_seed': 14, 'colsample_bytree': 1.0, 'learning_rate': 0.1454893092, 'max_depth': -1, 'min_child_samples': 81,'categorical_feature':categorical_feature, 'num_leaves': 129, 'random_state': 42, 'reg_alpha': 0.1, 'reg_lambda': 0.30, 'subsample': 0.89, 'subsample_freq': 1, 'verbosity': -1}
else:
LGB_PARAMS = LGB_PARAMS_2JHN
cv_score = train_and_predict_for_one_coupling_type(
 coupling_type, submission, n_atoms=model_params[coupling_type], n_folds=N_FOLDS)
 cv_scores[coupling_type] = cv_score 
<save_to_csv>
def bert_tokenize(df, col):

 gs_folder_bert = "gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-12_H-768_A-12"
tf.io.gfile.listdir(gs_folder_bert)
 tokenizer = tokenization.FullTokenizer(
 vocab_file=os.path.join(gs_folder_bert, "vocab.txt"),
do_lower_case=True 
)
 bert_token =(lambda x: tokenizer 
.convert_tokens_to_ids(['[CLS]'] + tokenizer.tokenize(x)+ ['[SEP]'])
)
 num_unique_words = len(tokenizer.vocab)
 pre_words_ids = df[col].apply(bert_token)
 pre_masks = df[col].apply(lambda x: [1]*(len(tokenizer.tokenize(x)) +2))
words_ids = tf.keras.preprocessing.sequence.pad_sequences(pre_words_ids)
 masks = tf.keras.preprocessing.sequence.pad_sequences(pre_masks)
 type_ids = np.zeros(words_ids.shape, dtype=np.int)
 word_cols = [f"{text}_{i:02d}" for i in range(words_ids.shape[1])]
mask_cols = [f"mask_{i:02d}"
for i in range(words_ids.shape[1])
 ]
type_cols = [f"type_{i:02d}"
for i in range(words_ids.shape[1])
 ]
df[word_cols] = words_ids
df[mask_cols] = masks
df[type_cols] = type_ids
return tokenizer, num_unique_words, word_cols, mask_cols, type_cols
submission.to_csv(f'{SUBMISSIONS_PATH}/submission.csv' )<define_variables>
def tf_tokenizer(df, col, num_words):

 tokenizer =(
 tf.keras.preprocessing.text.Tokenizer(num_words=num_words)
)
 tokenizer.fit_on_texts(df[col].values)
 word_ar = tf.keras.preprocessing.sequence.pad_sequences(
 tokenizer.texts_to_sequences(df[col].values)
)
 word_cols = [f"text_{i:02d}" for i in range(word_ar.shape[1])]
df[word_cols] = word_ar
return tokenizer, num_words, word_cols
DATA_PATH = '.. /input'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}<set_options>
df_full = pd.concat([df_train, df_test], ignore_index=True)
 tokenizer, num_unique_words, word_cols, mask_cols, type_cols = bert_tokenize(df_full, text)
 log.info(f"Vocab size: {num_unique_words}" )
%matplotlib inline 
<set_options>
hub_url_bert = "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2"
bert_layer = hub.KerasLayer(hub_url_bert, trainable=True )
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 120)
 pd.set_option('display.max_columns', 120 )<load_from_csv>
class TwolayerModel(tf.keras.Model):

 def __init__(self,
batch_size=32,
units=40,
embed_dim=100,
sequence_length=len(word_cols), 
):

 super(TwolayerModel, self ).__init__()
self.inps =(None, sequence_length)
 self.bs = batch_size
out_dim = 2
self._embed1 = tf.keras.layers.Embedding(
 num_unique_words,
embed_dim,
input_length=self.inps[1],
name="word_embedding", 
)
 self._lstm1 = tf.keras.layers.Bidirectional(
 tf.keras.layers.GRU(
 units,
name="lstm1",
return_sequences=True, 
)
)
 self._lstm2 = tf.keras.layers.Bidirectional(
 tf.keras.layers.GRU(
 units,
name="lstm2", 
)
)
 self._dense2 = tf.keras.layers.Dense(
 out_dim,
activation=tf.nn.softmax,
name="final", 
)
 self._optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001)
 self._metrics = [tf.keras.metrics.BinaryAccuracy() ]
self._loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
 self.compile(
 optimizer=self._optimizer,
loss=self._loss,
metrics=self._metrics, 
)
 def call(self, inputs):
x1 = self._embed1(inputs)
 y1 = self._lstm1(x1)
 y1 = self._lstm2(y1)
 out = self._dense2(y1)
 return out
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 train_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]
train_csv.head(10 )<load_from_csv>
class OnelayerModel(tf.keras.Model):

 def __init__(self,
batch_size=32,
units=40,
embed_dim=100,
sequence_length=len(word_cols), 
):

 super(OnelayerModel, self ).__init__()
self.inps = [ 
(None, sequence_length), 
(None, sequence_length), 
(None, sequence_length),
]
self.bs = batch_size
out_dim = 2
self._embed1 = tf.keras.layers.Embedding(
 num_unique_words,
embed_dim,
input_length=self.inps[0][1],
name="word_embedding", 
)
 self._lstm1 = tf.keras.layers.Bidirectional(
 tf.keras.layers.GRU(
 units,
name="lstm1", 
)
)
 self._dense2 = tf.keras.layers.Dense(
 out_dim,
activation=tf.nn.softmax,
name="final", 
)
 self._optimizer = tf.keras.optimizers.Adam(
 learning_rate=0.0001 
)
 self._metrics = [tf.keras.metrics.BinaryAccuracy() ]
self._loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
 self.compile(
 optimizer=self._optimizer,
loss=self._loss,
metrics=self._metrics, 
)
 def call(self, inputs):
x1 = self._embed1(inputs)
 y1 = self._lstm1(x1)
 out = self._dense2(y1)
 return out
submission_csv = pd.read_csv(f'{DATA_PATH}/sample_submission.csv', index_col='id' )<load_from_csv>
class ConvModel(tf.keras.Model):

 def __init__(self,
batch_size=32,
units=40,
embed_dim=100,
sequence_length=len(word_cols), 
):

 self.inps =(None, sequence_length)
 self.bs = batch_size
out_dim = 2
super(ConvModel, self ).__init__()
self._embed1 = tf.keras.layers.Embedding(
 num_unique_words,
embed_dim,
input_length=self.inps[1],
name="word_embedding", 
)
 filters = 100
window = 5
self._conv1 = tf.keras.layers.Conv1D(
 filters,
window 
)
 self._flatten = tf.keras.layers.Flatten()
self._dense1 = tf.keras.layers.Dense(
 units,
activation=tf.nn.relu,
name="dense", 
)
 self._dense2 = tf.keras.layers.Dense(
 out_dim,
activation=tf.nn.softmax,
name="final", 
)
 self._optimizer = tf.keras.optimizers.Adam(
 learning_rate=0.0001 
)
 self._metrics = [tf.keras.metrics.BinaryAccuracy() ]
self._loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
 self.compile(
 optimizer=self._optimizer,
loss=self._loss,
metrics=self._metrics, 
)
 def call(self, inputs):
x1 = self._embed1(inputs)
 x1 = self._conv1(x1)
 y1 = self._flatten(x1)
 z = self._dense1(y1)
 out = self._dense2(z)
 return out
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 test_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]
test_csv.head(10 )<data_type_conversions>
class BERTModel(tf.keras.Model):

 def __init__(self,
batch_size=64,
units=40,
embed_dim=100,
sequence_length=len(word_cols), 
):

 super(BERTModel, self ).__init__()
self.inps = [ 
(None, sequence_length), 
(None, sequence_length), 
(None, sequence_length),
]
self.bs = batch_size
out_dim = 2
self.max_seq_length = sequence_length
self.bert_layer = bert_layer
self.input_word_ids = tf.keras.Input(
 shape=(self.max_seq_length,),
dtype=tf.int32,
name='input_word_ids', 
)
 self.input_m = tf.keras.Input(
 shape=(self.max_seq_length,),
dtype=tf.int32,
name='input_mask', 
)
 self.segment_ids = tf.keras.Input(
 shape=(self.max_seq_length,),
dtype=tf.int32,
name='segment_ids', 
)
 self._dense0 = tf.keras.layers.Dense(units, activation=tf.nn.relu)
 self._do0 = tf.keras.layers.Dropout(0.5)
 self._dense2 = tf.keras.layers.Dense(
 out_dim,
activation=tf.nn.softmax,
name="final", 
)
 self._optimizer = tf.keras.optimizers.Adam(learning_rate=0.00001)
 self._metrics = [tf.keras.metrics.BinaryAccuracy() ]
self._loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
 self.compile(
 optimizer=self._optimizer,
loss=self._loss,
metrics=self._metrics, 
)
 self.inputs =(self.input_word_ids, self.input_m, self.segment_ids)
 self.outputs = self._dense2
def call(self, inputs, training=False):
self.input_word_ids, self.input_m, self.segment_ids = inputs
pooled_output, sequence_output = self.bert_layer(
 [
self.input_word_ids,
self.input_m,
self.segment_ids,
] 
)
 clf_output = sequence_output[:, 0, :]
z = self._dense0(clf_output)
 if training:
z = self._do0(z, training=training)
 out = self._dense2(z)
 return out
structures_dtypes = {
'molecule_name': 'category',
'atom_index': 'int8',
'atom': 'category',
'x': 'float32',
'y': 'float32',
'z': 'float32'
}
structures_csv = pd.read_csv(f'{DATA_PATH}/structures.csv', dtype=structures_dtypes)
 structures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 structures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]
structures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS ).astype('int8')
 structures_csv.head(10 )<data_type_conversions>
tfboard_dir = "logs"
if not os.path.exists(tfboard_dir):
os.mkdir(tfboard_dir)
 tensorboard_callback = tf.keras.callbacks.TensorBoard(
 log_dir=tfboard_dir,
histogram_freq=1,
write_graph=True,
write_images=True, 
)
 early_stopping = tf.keras.callbacks.EarlyStopping(
 monitor="val_binary_accuracy",
min_delta=1e-5,
patience=10,
baseline=0.5,
restore_best_weights=True,
)
def build_type_dataframes(base, structures, coupling_type):
base = base[base['type'] == coupling_type].drop('type', axis=1 ).copy()
base = base.reset_index()
base['id'] = base['id'].astype('int32')
 structures = structures[structures['molecule_index'].isin(base['molecule_index'])]
return base, structures<merge>
model = BERTModel(batch_size=32)
 df_test = df_full.iloc[train_pts:]
df_train = df_full.iloc[:train_pts]
log.info(f"Dataset size: {df_train.shape[0]}")
 remainder = df_train.shape[0] % model.bs
pad_size = model.bs - remainder if remainder !=0 else 0
log.info(f"Remainder from batch size: {remainder}
"
f"Padding {pad_size} elements." 
)
 X1 = df_train[word_cols].values.astype(int)
 X2 = df_train[mask_cols].values.astype(int)
 X3 = df_train[type_cols].values.astype(int)
 X_unpad =(
 tf.convert_to_tensor(X1, dtype=tf.dtypes.int32, name='input_word_ids'),
tf.convert_to_tensor(X2, dtype=tf.dtypes.int32, name='input_mask'),
tf.convert_to_tensor(X3, dtype=tf.dtypes.int32, name='input_type_ids'), 
)
 if pad_size > 0:
X1 = np.vstack([X1, np.zeros(( pad_size, X1.shape[1])).astype(int)])
 X2 = np.vstack([X2, np.zeros(( pad_size, X1.shape[1])).astype(int)])
 X3 = np.vstack([X3, np.zeros(( pad_size, X1.shape[1])).astype(int)])
 X1 = tf.convert_to_tensor(X1, dtype=tf.dtypes.int32, name='input_word_ids')
 X2 = tf.convert_to_tensor(X2, dtype=tf.dtypes.int32, name='input_mask')
 X3 = tf.convert_to_tensor(X3, dtype=tf.dtypes.int32, name='input_type_ids')
 df_train[y_cols] = pd.get_dummies(df_train[target], prefix=target)
 Y = df_train[y_cols].values
Y_add = np.zeros(( pad_size, 2))
Y_add[:, 0] = 1.0
if pad_size > 0:
Y = np.vstack([Y, Y_add])
 X_val =(X1, X2, X3)
 X =(
 tf.data.Dataset.from_tensor_slices(( X_val, Y)) 
.batch(model.bs, drop_remainder=True)
)
 X = X.shuffle(buffer_size=10000)
 q = 5
p = 1
select = lambda x, y:(x % q <= p)
 nselect = lambda x, y: ~(x % q <= p)
 take = lambda x, y: y
X_train = X.enumerate().filter(nselect ).map(take)
 X_valid = X.enumerate().filter(select ).map(take)
 log.info(f"Test dataset size: {df_test.shape}")
 Z1 = df_test[word_cols].values.astype(int)
 Z2 = df_test[mask_cols].values.astype(int)
 Z3 = df_test[type_cols].values.astype(int)
 Z1 = tf.convert_to_tensor(Z1, dtype=tf.int32, name='input_word_ids')
 Z2 = tf.convert_to_tensor(Z2, dtype=tf.int32, name='input_mask')
 Z3 = tf.convert_to_tensor(Z3, dtype=tf.int32, name='input_type_ids')
 X_test =(Z1, Z2, Z3 )
def add_coordinates(base, structures, index):
df = pd.merge(base, structures, how='inner',
left_on=['molecule_index', f'atom_index_{index}'],
right_on=['molecule_index', 'atom_index'] ).drop(['atom_index'], axis=1)
 df = df.rename(columns={
'atom': f'atom_{index}',
'x': f'x_{index}',
'y': f'y_{index}',
'z': f'z_{index}'
})
 return df<merge>
hist = model.fit(
 X_train,
epochs=8,
validation_data=X_valid,
callbacks=[
early_stopping,
],
)
def add_atoms(base, atoms):
df = pd.merge(base, atoms, how='inner',
on=['molecule_index', 'atom_index_0', 'atom_index_1'])
 return df<merge>
Y_train_pred = model.predict(X_unpad)
 Y_test_pred = model.predict(X_test )
def merge_all_atoms(base, structures):
df = pd.merge(base, structures, how='left',
left_on=['molecule_index'],
right_on=['molecule_index'])
 df = df[(df.atom_index_0 != df.atom_index)&(df.atom_index_1 != df.atom_index)]
return df<feature_engineering>
df_train_pred = pd.DataFrame(Y_train_pred, columns=y_cols)
 df_train_pred = df_train_pred.apply(np.round ).astype({x: int for x in y_cols})
 df_train_pred[target] = df_train_pred["target_1"]
df_train_pred.drop(y_cols, inplace=True, axis=1)
 df_train_pred["id"] = df_train["id"].values
df_train_pred = df_train_pred[["id", target]]
def add_center(df):
df['x_c'] =(( df['x_1'] + df['x_0'])* np.float32(0.5))
df['y_c'] =(( df['y_1'] + df['y_0'])* np.float32(0.5))
df['z_c'] =(( df['z_1'] + df['z_0'])* np.float32(0.5))
def add_distance_to_center(df):
df['d_c'] =(( 
(df['x_c'] - df['x'])**np.float32(2)+ 
(df['y_c'] - df['y'])**np.float32(2)+ 
(df['z_c'] - df['z'])**np.float32(2)
)**np.float32(0.5))
def add_distance_between(df, suffix1, suffix2):
df[f'd_{suffix1}_{suffix2}'] =(( 
(df[f'x_{suffix1}'] - df[f'x_{suffix2}'])**np.float32(2)+ 
(df[f'y_{suffix1}'] - df[f'y_{suffix2}'])**np.float32(2)+ 
(df[f'z_{suffix1}'] - df[f'z_{suffix2}'])**np.float32(2)
)**np.float32(0.5))<merge>
df_test_pred = pd.DataFrame(Y_test_pred, columns=y_cols)
 df_test_pred = df_test_pred.apply(np.round ).astype({x: int for x in y_cols})
 df_test_pred[target] = df_test_pred["target_1"]
df_test_pred.drop(y_cols, inplace=True, axis=1)
 df_test_pred.drop(list(df_test_pred.index[df_train.shape[0]:]), inplace=True, axis=0)
 df_test_pred["id"] = df_test["id"].values
df_test_pred = df_test_pred[["id", target]]
def add_n_atoms(base, structures):
dfs = structures['molecule_index'].value_counts().rename('n_atoms' ).to_frame()
return pd.merge(base, dfs, left_on='molecule_index', right_index=True )<define_variables>
log.info("
" +
sklearn.metrics.classification_report(
 df_train[target],
df_train_pred[target],
target_names=["Not disaster", "Disaster"] 
)
 )
def take_n_atoms(df, n_atoms, four_start=4):
labels = []
for i in range(2, n_atoms):
label = f'atom_{i}'
labels.append(label)
 for i in range(n_atoms):
num = min(i, 4)if i < four_start else 4
for j in range(num):
labels.append(f'd_{i}_{j}')
 if 'scalar_coupling_constant' in df:
labels.append('scalar_coupling_constant')
 return df[labels]<create_dataframe>
log.info("Training accuracy score {}.".format(
 sklearn.metrics.accuracy_score(df_train[target], df_train_pred[target])
)
 )
<train_model><EOS> 
output_dir = "./"
results_bn = "results.csv"
results_fn = os.path.join(output_dir, results_bn)
 df_test_pred.to_csv(results_fn, index=False )