Kaggle competition: Mechanisms of Action (MoA) Prediction
%%time
print("Vectorizing data...")
 default_preprocessor = CountVectorizer().build_preprocessor()
def build_preprocessor(field):
field_idx = list(full_df.columns ).index(field)
 return lambda x: default_preprocessor(x[field_idx])
 vectorizer = FeatureUnion([ 
('name', CountVectorizer(
 ngram_range=(1, 2),
max_features=50000,
preprocessor=build_preprocessor('name'))), 
('category_name', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('category_name'))), 
('brand_name', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('brand_name'))), 
('shipping', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('shipping'))), 
('item_condition_id', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('item_condition_id'))), 
('item_description', TfidfVectorizer(
 ngram_range=(1, 3),
max_features=100000,
preprocessor=build_preprocessor('item_description'))),
])
 X = vectorizer.fit_transform(full_df.values)
 X_train = X[:n_trains]
X_dev = X[n_trains:n_trains+n_devs]
X_test = X[n_trains+n_devs:]
print(X.shape, X_train.shape, X_dev.shape, X_test.shape )<train_model>
import pickle
import numpy as np
import pandas as pd
import tensorflow as tf
%%time
print("Fitting Ridge model on training examples...")
 ridge_model = Ridge(
 solver='auto', fit_intercept=True, alpha=0.5,
max_iter=100, normalize=False, tol=0.05, 
)
 ridge_model.fit(X_train, Y_train )<predict_on_test>
def build_callbacks(
 model_path, factor=0.1, mode="auto", monitor="val_loss", patience=0, verbose=0 
):
early_stopping = tf.keras.callbacks.EarlyStopping(
 mode=mode, monitor=monitor, patience=patience, verbose=verbose 
)
 model_checkpoint = tf.keras.callbacks.ModelCheckpoint(
 model_path, mode=mode, monitor=monitor, save_best_only=True, verbose=verbose 
)
 reduce_lr_on_plateau = tf.keras.callbacks.ReduceLROnPlateau(
 factor=factor, monitor=monitor, mode=mode, verbose=verbose 
)
 return [early_stopping, model_checkpoint, reduce_lr_on_plateau]
Y_dev_preds_ridge = ridge_model.predict(X_dev)
 Y_dev_preds_ridge = Y_dev_preds_ridge.reshape(-1, 1)
 print("RMSL error on dev set:", rmsle(Y_dev, Y_dev_preds_ridge))<predict_on_test>
def build_mlp_classifier(
 input_dim,
output_dim,
activation="relu",
bias_initializer="zeros",
kernel_initializer="glorot_uniform",
label_smoothing=0.0,
lr=1e-03,
momentum=0.99,
n_layers=3,
n_units="auto",
pretrained_model_path=None,
rate=0.0,
skip=False, 
):
if n_units == "auto":
n_units = 0.5 *(input_dim + output_dim)
 inputs = tf.keras.layers.Input(shape=input_dim, name="I")
 x = inputs
for i in range(n_layers - 2):
x = tfa.layers.WeightNormalization(
 tf.keras.layers.Dense(n_units, kernel_initializer=kernel_initializer),
name=f"WN{i + 1}",
)(x)
 x = tf.keras.layers.BatchNormalization(momentum=momentum, name=f"BN{i + 1}" )(x)
 if skip and i > 0 and i % 2 == 0:
x = x + shortcut
x = tf.keras.layers.Activation(activation, name=f"A{i + 1}" )(x)
 x = tf.keras.layers.Dropout(rate, name=f"D{i + 1}" )(x)
 if skip and i % 2 == 0:
shortcut = x
x = tfa.layers.WeightNormalization(
 tf.keras.layers.Dense(output_dim, bias_initializer=bias_initializer)
 )(x)
 outputs = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inputs, outputs=outputs)
 if pretrained_model_path is not None:
model.load_weights(pretrained_model_path, by_name=True)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing)
 optimizer = AdaBeliefOptimizer(learning_rate=lr)
 model.compile(loss=loss, optimizer=optimizer)
 return model
%%time
ridge_preds = ridge_model.predict(X_test)
 ridge_preds = np.expm1(ridge_preds )<compute_train_metric>
def compute_row_statistics(X, prefix=""):
Xt = pd.DataFrame()
for agg_func in [
"mean",
"std",
"kurtosis",
"skew",
]:
Xt[f"{prefix}{agg_func}"] = X.agg(agg_func, axis=1)
 return Xt
def aggregate_predicts(Y1, Y2):
assert Y1.shape == Y2.shape
ratio = 0.63
return Y1 * ratio + Y2 *(1.0 - ratio)
 Y_dev_preds = aggregate_predicts(Y_dev_preds_rnn, Y_dev_preds_ridge)
 print("RMSL error for RNN + Ridge on dev set:", rmsle(Y_dev, Y_dev_preds))<save_to_csv>
preds = aggregate_predicts(rnn_preds, ridge_preds)
 submission = pd.DataFrame({
"test_id": test_df.test_id,
"price": preds.reshape(-1),
})
 submission.to_csv("./rnn_ridge_submission.csv", index=False )<import_modules>
def score(Y, Y_pred, eps=1e-15, label_smoothing=0.0):
Y = np.asarray(Y)
 Y = np.ravel(Y)
 if label_smoothing > 0.0:
Y = Y *(1.0 - label_smoothing)+ 0.5 * label_smoothing
Y_pred = np.asarray(Y_pred)
 Y_pred = np.ravel(Y_pred)
 Y_pred = np.clip(Y_pred, eps, 1.0 - eps)
 return -np.mean(Y * np.log(Y_pred)+(1.0 - Y)* np.log(1.0 - Y_pred))
import eli5
import nltk
import numpy as np
import pandas as pd
from sklearn.model_selection import KFold
from sklearn.linear_model import Ridge
from sklearn.pipeline import FeatureUnion
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics import mean_squared_log_error
from nltk.stem.porter import PorterStemmer
import gc
import re<load_from_csv>
def set_seed(seed=0):
os.environ["PYTHONHASHSEED"] = str(seed)
 rn.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 graph = tf.compat.v1.get_default_graph()
session_conf = tf.compat.v1.ConfigProto(
 inter_op_parallelism_threads=1, intra_op_parallelism_threads=1 
)
 sess = tf.compat.v1.Session(graph=graph, config=session_conf)
 tf.compat.v1.keras.backend.set_session(sess )
train = pd.read_table('.. /input/train.tsv')
 test = pd.read_table('.. /input/test.tsv')
 STOP_WORDS = frozenset([
"a", "about", "after", "afterwards", "again",
"all", "almost", "along", "already", "also", "although",
"am", "among", "amongst", "amoungst", "an", "and", "another",
"any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are",
"as", "at", "back", "be", "became", "because", "become",
"becomes", "becoming", "been", "before","behind", "being",
"beside", "between", "both",
"but", "by", "can", "cannot", "cant", "co", "con",
"could", "couldnt", "de", "do",
"each", "eg", "eight", "either", "else",
"elsewhere", "etc", "even", "ever", "every",
"everything", "everywhere", "few",
"find", "for",
"from", "go",
"had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter",
"hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his",
"how", "however", "i", "ie", "if", "in"
"into", "is", "it", "its", "itself",
"latterly", "ltd", "many", "may", "me",
"meanwhile", "might", "mill", "mine","moreover",
"my", "myself","neither",
"never", "nevertheless", "no", "nobody", "none", "noone",
"nor", "not", "now", "of", "on",
"once", "one", "only", "onto", "or", "other", "others", "otherwise", "our",
"ours", "ourselves", "out", "per", "perhaps",
"re",
"seeming", "seems", "she",
"since", "so", "some", "somehow", "someone",
"something", "sometime", "sometimes", "somewhere",
"that", "the", "their", "them",
"themselves", "then", "thence", "there", "thereafter", "thereby",
"therefore", "therein", "thereupon", "these", "they",
"this", "those", "though",
"thru", "thus", "to", "together", "too", "toward", "towards",
"twelve", "un", "until", "up", "upon", "us",
"via", "was", "we", "were", "what", "when",
"whence", "where", "whereafter", "whereas", "whereby",
"wherein", "whereupon", "wherever", "whether", "which", "while", "whither",
"who", "whoever", "whom", "whose", "why", "with",
"within", "without", "would", "yet", "you", "your", "yours", "yourself",
"yourselves"])
 NAME_MIN_DF = 10
MAX_FEATURES_ITEM_NAME = 50000
MAX_FEATURES_ITEM_DESCRIPTION = 100000
transformerWeights={
'name': 1.0,
'general_cat': 1.0,
'subcat_1': 1.0,
'subcat_2': 1.0,
'brand_name': 1.2,
'shipping': 1.0,
'item_condition_id': 1.0,
'len_description': 1.0,
'item_description': 0.8
}
train.drop(train[train.price < 1.0].index, inplace=True)
 train = train.reset_index(drop=True)
 nrow_train = train.shape[0]
train_test : pd.DataFrame = pd.concat([train, test])
 y_train = np.log1p(train['price'])
 del train
gc.collect()<data_type_conversions>
class ClippedFeatures(BaseEstimator, TransformerMixin):
def __init__(self, copy=True, high=0.99, low=0.01):
self.copy = copy
self.high = high
self.low = low
def fit(self, X, y=None):
self.data_max_ = X.quantile(q=self.high)
 self.data_min_ = X.quantile(q=self.low)
 return self
def transform(self, X):
if self.copy:
X = X.copy()
X.clip(self.data_min_, self.data_max_, axis=1, inplace=True)
 return X
train_test['category_name'] = train_test['category_name'].fillna('Other' ).astype(str)
 train_test['brand_name'] = train_test['brand_name'].fillna('missing' ).astype(str)
 train_test['shipping'] = train_test['shipping'].astype(str)
 train_test['item_condition_id'] = train_test['item_condition_id'].astype(str)
 train_test['item_description'] = train_test['item_description'].fillna('[ndy]')
 def replace_text(df, variable, text_to_replace, replacement):
df.loc[df[variable] == text_to_replace, variable] = replacement
replace_text(train_test, 'item_description', 'No description yet', '[ndy]')
 def wordCount(text):
try:
text = text.lower()
regex = re.compile('[' +re.escape(string.punctuation)+ '0-9\\r\\t\
]')
 txt = regex.sub(" ", text)
 words = [w for w in txt.split(" ")\
if not w in stop_words.ENGLISH_STOP_WORDS and len(w)>3]
if len(words)< 30:
return "1"
elif len(words)< 90:
return "2"
elif len(words)< 120:
return "3"
else:
return "4"
except:
return "1"
train_test['len_description'] = train_test['item_description'].apply(lambda x: wordCount(x))
train_test['general_cat'], train_test['subcat_1'], train_test['subcat_2'] = train_test['category_name'].str.split("/", 2 ).str
train_test.drop('category_name', axis=1, inplace=True)
 train_test['general_cat'] = train_test['general_cat'].fillna('Other' ).astype(str)
 train_test['subcat_1'] = train_test['subcat_1'].fillna('Other' ).astype(str)
 train_test['subcat_2'] = train_test['subcat_2'].fillna('Other' ).astype(str )<feature_engineering>
class Cutmix(tf.keras.utils.Sequence):
def __init__(self, X, y=None, batch_size=32, alpha=1.0):
self.X = np.asarray(X)
 if y is None:
self.y = y
else:
self.y = np.asarray(y)
 self.batch_size = batch_size
self.alpha = alpha
def __getitem__(self, i):
X_batch = self.X[i * self.batch_size :(i + 1)* self.batch_size]
n_samples, n_features = self.X.shape
batch_size = X_batch.shape[0]
shuffle = np.random.choice(n_samples, batch_size)
 l = np.random.beta(self.alpha, self.alpha)
 mask = np.random.choice([0.0, 1.0], size=n_features, p=[1.0 - l, l])
 X_shuffle = self.X[shuffle]
X_batch = mask * X_batch +(1.0 - mask)* X_shuffle
if self.y is None:
return X_batch, None
y_batch = self.y[i * self.batch_size :(i + 1)* self.batch_size]
y_shuffle = self.y[shuffle]
y_batch = l * y_batch +(1.0 - l)* y_shuffle
return X_batch, y_batch
def __len__(self):
n_samples = self.X.shape[0]
return int(np.ceil(n_samples / self.batch_size))
%%time
default_preprocessor = CountVectorizer().build_preprocessor()
def build_preprocessor(field):
field_idx = list(train_test.columns ).index(field)
 return lambda x: default_preprocessor(x[field_idx])
 def rex_tokenizer(text):
token_pattern = re.compile(r"(?u)\b\w[\w-]*\w\b")
 tokens = token_pattern.findall(text)
 item_list = {"1tb" : "1 tb", "2tb" : "2 tb", "4tb" : "4 tb", "4g" : "4 gb","4gb" : "4 gb","8g" : "8 gb","8gb" : "8 gb","16g" : "16 gb", "16gb" : "16 gb", "32gb" : "32 gb", "32g" : "32 gb","64gb" : "64 gb", "64g" : "64 gb", "64gb" : "64 gb", "80gb" : "80 gb", "120gb" : "128 gb", "128gb" : "128 gb", "128g" : "128 gb",
"160gb" : "160 gb", "250gb" : "256 gb", "256gb" : "256 gb",
"500g" : "512 gb", "500gb" : "512 gb", "512g" : "512 gb","512gb" : "512 gb", "10k" : "10 k", "10kt" : "10 k", "12k" : "12 k","14k" : "14 k", "14kt" : "14 k" , "18k" : "18 k" ,"18kt" : "18 k" , "1oz" : "1 oz", "4oz" : "4 oz", "5oz" : "5 oz", "8oz" : "8 oz","36oz" : "36 oz", "64oz" : "64 oz"}
postTokens = []
for item in tokens:
if item in item_list:
item = item_list[item]
postTokens.append(item)
 return postTokens
vectorizer = FeatureUnion([ 
('name', CountVectorizer(
 ngram_range=(1, 2),
min_df=NAME_MIN_DF,
tokenizer=rex_tokenizer,
stop_words = 'english',
preprocessor=build_preprocessor('name'))), 
('general_cat', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('general_cat'))), 
('subcat_1', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('subcat_1'))), 
('subcat_2', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('subcat_2'))), 
('brand_name', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('brand_name'))), 
('shipping', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('shipping'))), 
('item_condition_id', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('item_condition_id'))), 
('item_description', TfidfVectorizer(
 ngram_range=(1, 3),
max_features=MAX_FEATURES_ITEM_DESCRIPTION,
stop_words = 'english',
analyzer = 'word',
tokenizer=rex_tokenizer,
preprocessor=build_preprocessor('item_description'))), 
('len_description', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('len_description'))),
], transformer_weights=transformerWeights)
 X_train_test = vectorizer.fit_transform(train_test.values )<compute_train_metric>
%%time
def get_rmsle(y_true, y_pred):
return np.sqrt(mean_squared_log_error(np.expm1(y_true), np.expm1(y_pred)))
 cv = KFold(n_splits=10, shuffle=True, random_state=42)
 for train_ids, valid_ids in cv.split(X_train_test[:nrow_train]):
model = Ridge(
 solver='auto',
fit_intercept=True,
alpha=0.5,
max_iter=100,
normalize=False,
copy_X=True,
random_state=101,
tol=0.025)
 model.fit(X_train_test[train_ids], y_train[train_ids])
 y_pred_valid = model.predict(X_train_test[valid_ids])
 rmsle = get_rmsle(y_pred_valid, y_train[valid_ids])
 print(f'valid rmsle: {rmsle:.5f}' )<load_pretrained>
eli5.show_weights(model, vec=vectorizer, top=100, feature_filter=lambda x: x != '<BIAS>' )<save_to_csv>
class MultilabelStratifiedGroupKFold(_BaseKFold):
def __init__(self, n_splits=5, random_state=None, shuffle=False):
super().__init__(n_splits=n_splits, random_state=random_state, shuffle=shuffle)
 def _iter_test_indices(self, X=None, y=None, groups=None):
cv = MultilabelStratifiedKFold(
 n_splits=self.n_splits,
random_state=self.random_state,
shuffle=self.shuffle, 
)
 value_counts = groups.value_counts()
regluar_indices = value_counts.loc[ 
(value_counts == 6)|(value_counts == 12)|(value_counts == 18)
 ].index.sort_values()
irregluar_indices = value_counts.loc[ 
(value_counts != 6)&(value_counts != 12)&(value_counts != 18)
 ].index.sort_values()
group_to_fold = {}
tmp = y.groupby(groups ).mean().loc[regluar_indices]
for fold,(_, test)in enumerate(cv.split(tmp, tmp)) :
group_to_fold.update({group: fold for group in tmp.index[test]})
 sample_to_fold = {}
tmp = y.loc[groups.isin(irregluar_indices)]
for fold,(_, test)in enumerate(cv.split(tmp, tmp)) :
sample_to_fold.update({sample: fold for sample in tmp.index[test]})
 folds = groups.map(group_to_fold)
 is_na = folds.isna()
folds[is_na] = folds[is_na].index.map(sample_to_fold ).values
for i in range(self.n_splits):
yield np.where(folds == i)[0]
preds = model.predict(X_train_test[nrow_train:])
 test["price"] = np.expm1(preds)
 test[["test_id", "price"]].to_csv("submission_ridge.csv", index = False )<feature_engineering>
dtype = {"cp_type": "category", "cp_dose": "category"}
index_col = "sig_id"
train_features = pd.read_csv(
 ".. /input/lish-moa/train_features.csv", dtype=dtype, index_col=index_col 
)
 X = train_features.select_dtypes("number")
 Y = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv", index_col=index_col)
 groups = pd.read_csv(
 ".. /input/lish-moa/train_drug.csv", index_col=index_col, squeeze=True 
)
 columns = Y.columns
def rmsle(y, y0):
assert len(y)== len(y0)
 return np.sqrt(np.mean(np.power(np.log1p(y)-np.log1p(y0), 2)))
 def split_cat(text):
try: return text.split("/")
 except: return("No Label", "No Label", "No Label")
 def handle_missing_inplace(dataset):
dataset['general_cat'].fillna(value='missing', inplace=True)
 dataset['subcat_1'].fillna(value='missing', inplace=True)
 dataset['subcat_2'].fillna(value='missing', inplace=True)
 dataset['brand_name'].fillna(value='missing', inplace=True)
 dataset['item_description'].fillna(value='missing', inplace=True)
 NUM_BRANDS = 4809
NUM_CATEGORIES = 1287
def cutting(dataset):
pop_brand = dataset['brand_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]
dataset.loc[~dataset['brand_name'].isin(pop_brand), 'brand_name'] = 'missing'
pop_category1 = dataset['general_cat'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
pop_category2 = dataset['subcat_1'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
pop_category3 = dataset['subcat_2'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
dataset.loc[~dataset['general_cat'].isin(pop_category1), 'general_cat'] = 'missing'
dataset.loc[~dataset['subcat_1'].isin(pop_category2), 'subcat_1'] = 'missing'
dataset.loc[~dataset['subcat_2'].isin(pop_category3), 'subcat_2'] = 'missing'
def to_categorical(dataset):
dataset['general_cat'] = dataset['general_cat'].astype('category')
 dataset['subcat_1'] = dataset['subcat_1'].astype('category')
 dataset['subcat_2'] = dataset['subcat_2'].astype('category')
 dataset['item_condition_id'] = dataset['item_condition_id'].astype('category' )<load_from_csv>
c_prefix = "c-"
g_prefix = "g-"
c_columns = X.columns.str.startswith(c_prefix)
 g_columns = X.columns.str.startswith(g_prefix)
 X_stats_c = compute_row_statistics(X.loc[:, c_columns], prefix=c_prefix)
 X_stats_g = compute_row_statistics(X.loc[:, g_columns], prefix=g_prefix)
 with open(".. /input/preprocessor-fit/clipped_features.pkl", "rb")as f:
clipped_features = pickle.load(f)
 X = clipped_features.transform(X)
 X = pd.concat([X, X_stats_c, X_stats_g], axis=1)
start_time = time.time()
train = pd.read_table('.. /input/train.tsv', engine='c')
 test = pd.read_table('.. /input/test.tsv', engine='c')
 print('[{}] Finished to load data'.format(time.time() - start_time))<prepare_x_and_y>
alpha = 4.0
batch_size = 32
factor = 0.5
n_seeds = 5
n_splits = 5
patience = 30
shuffle = True
params = {
"activation": "elu",
"kernel_initializer": "he_normal",
"label_smoothing": 5e-04,
"lr": 0.03,
"n_layers": 6,
"n_units": 256,
"rate": 0.3,
}
fit_params = {"epochs": 1_000, "verbose": 0}
nrow_train = train.shape[0]
y = np.log1p(train["price"])
 merge: pd.DataFrame = pd.concat([train, test])
 submission: pd.DataFrame = test[['test_id']]<feature_engineering>
start_time = time.time()
merge['general_cat'], merge['subcat_1'], merge['subcat_2'] = \
zip(*merge['category_name'].apply(lambda x: split_cat(x)))
 merge.drop('category_name', axis=1, inplace=True)
 print('[{}] Split categories completed.'.format(time.time() - start_time))<count_missing_values>
%%time
bias_initializer = -Y.mean(axis=0 ).apply(np.log ).values
bias_initializer = tf.keras.initializers.Constant(bias_initializer)
 Y_pred = np.zeros(( train_size, n_classes))
Y_pred = pd.DataFrame(Y_pred, columns=Y.columns, index=Y.index)
 for i in range(n_seeds):
set_seed(seed=i)
 cv = MultilabelStratifiedGroupKFold(
 n_splits=n_splits, random_state=i, shuffle=shuffle 
)
 for j,(train, valid)in enumerate(cv.split(X, Y[columns], groups)) :
model_path = f"model_seed_{i}_fold_{j}.h5"
model = build_mlp_classifier(
 n_features,
n_classes,
bias_initializer=bias_initializer,
**params, 
)
 generator = Cutmix(
 X.iloc[train], Y.iloc[train], alpha=alpha, batch_size=batch_size 
)
 callbacks = build_callbacks(model_path, factor=factor, patience=patience)
 history = model.fit(
 generator,
callbacks=callbacks,
validation_data=(X.iloc[valid], Y.iloc[valid]),
**fit_params, 
)
 model.load_weights(model_path)
 Y_pred.iloc[valid] += model.predict(X.iloc[valid])/ n_seeds
Y_pred[train_features["cp_type"] == "ctl_vehicle"] = 0.0
with open("Y_pred.pkl", "wb")as f:
pickle.dump(Y_pred[columns], f )
<categorify><EOS> 
score(Y[columns], Y_pred[columns] )