Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<string_transform>
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 pd.set_option('max_columns', None)
tokenizer = TreebankWordTokenizer()
isolate_dict = {ord(c):f' {c} ' for c in symbols_to_isolate}
remove_dict = {ord(c):f'' for c in symbols_to_delete}
def handle_punctuation(x):
x = x.translate(remove_dict)
 x = x.translate(isolate_dict)
 return x
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def preprocess(x):
x = handle_punctuation(x)
 x = handle_contractions(x)
 x = fix_quote(x)
 return x<prepare_x_and_y>
%%time
X_train = pd.read_csv(os.path.join(location, 'X_train.csv'))
X_test = pd.read_csv(os.path.join(location, 'X_test.csv'))
y_train = pd.read_csv(os.path.join(location, 'y_train.csv'))
x_train = train['comment_text'].progress_apply(lambda x:preprocess(x))
y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = test['comment_text'].progress_apply(lambda x:preprocess(x))
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
max_features = 410047<string_transform>
print("Xtrain: {}
Xtest: {}
ytrain: {}".format(X_train.shape, X_test.shape, y_train.shape))
tokenizer = text.Tokenizer(num_words = max_features, filters='',lower=False )<train_model>
print('Size of the Xtrain')
 print('Numbers of Measurements: {0}
Numbers of columns: {1}'.format(X_train.shape[0], X_train.shape[1]))
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
max_features = max_features or len(tokenizer.word_index)+ 1
max_features
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()
y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32 )<string_transform>
print('Size of the Xtest')
 print('Numbers of Measurements: {0}
Numbers of columns: {1}'.format(X_test.shape[0], X_test.shape[1]))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )<categorify>
print('Size of the Labels')
 print('Numbers of Measurements: {0}
Numbers of columns: {1}'.format(y_train.shape[0], y_train.shape[1]))
lengths = torch.from_numpy(np.array([len(x)for x in x_train]))
maxlen = 300
x_train_padded = torch.from_numpy(sequence.pad_sequences(x_train, maxlen=maxlen))
x_train_padded.shape<categorify>
def missing_data(data):
total = data.isnull().sum()
percent =(total/data.isnull().count() *100)
 miss_column = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 miss_column['Types'] = types
return(np.transpose(miss_column))
class SequenceBucketCollator() :
def __init__(self, choose_length, sequence_index, length_index, label_index=None):
self.choose_length = choose_length
self.sequence_index = sequence_index
self.length_index = length_index
self.label_index = label_index
def __call__(self, batch):
batch = [torch.stack(x)for x in list(zip(*batch)) ]
sequences = batch[self.sequence_index]
lengths = batch[self.length_index]
length = self.choose_length(lengths)
 mask = torch.arange(start=maxlen, end=0, step=-1)< length
padded_sequences = sequences[:, mask]
batch[self.sequence_index] = padded_sequences
if self.label_index is not None:
return [x for i, x in enumerate(batch)if i != self.label_index], batch[self.label_index]
return batch<choose_model_class>
missing_data(X_train )
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x, lengths=None):
h_embedding = self.embedding(x.long())
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<categorify>
missing_data(X_test )
test_lengths = torch.from_numpy(np.array([len(x)for x in x_test]))
x_test_padded = torch.from_numpy(sequence.pad_sequences(x_test, maxlen=maxlen))<create_dataframe>
missing_data(y_train )
batch_size = 512
test_dataset = data.TensorDataset(x_test_padded, test_lengths)
 train_dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 valid_dataset = data.Subset(train_dataset, indices=[0, 1])
 train_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() ,
sequence_index=0,
length_index=1,
label_index=2)
 test_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() , sequence_index=0, length_index=1)
 train_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=train_collator)
 valid_loader = data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, collate_fn=train_collator)
 test_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=test_collator)
 databunch = DataBunch(train_dl=train_loader, valid_dl=valid_loader, collate_fn=train_collator )<compute_test_metric>
Surface_count = y_train['surface'].value_counts().reset_index().rename(columns = {'index' : 'Labels'})
 Surface_count
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2<choose_model_class>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
def data_engineering(actual):
new = pd.DataFrame()
actual['total_angular_velocity'] =(actual['angular_velocity_X'] ** 2 + actual['angular_velocity_Y'] ** 2 + actual['angular_velocity_Z'] ** 2)** 0.5
actual['total_linear_acceleration'] =(actual['linear_acceleration_X'] ** 2 + actual['linear_acceleration_Y'] ** 2 + actual['linear_acceleration_Z'] ** 2)** 0.5
actual['acc_vs_vel'] = actual['total_linear_acceleration'] / actual['total_angular_velocity']
x, y, z, w = actual['orientation_X'].tolist() , actual['orientation_Y'].tolist() , actual['orientation_Z'].tolist() , actual['orientation_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 actual['euler_x'] = nx
actual['euler_y'] = ny
actual['euler_z'] = nz
actual['total_angle'] =(actual['euler_x'] ** 2 + actual['euler_y'] ** 2 + actual['euler_z'] ** 2)** 5
actual['angle_vs_acc'] = actual['total_angle'] / actual['total_linear_acceleration']
actual['angle_vs_vel'] = actual['total_angle'] / actual['total_angular_velocity']
def f1(x):
return np.mean(np.diff(np.abs(np.diff(x))))
def f2(x):
return np.mean(np.abs(np.diff(x)))
 for col in actual.columns:
if col in ['row_id', 'series_id', 'measurement_number']:
continue
new[col + '_mean'] = actual.groupby(['series_id'])[col].mean()
new[col + '_min'] = actual.groupby(['series_id'])[col].min()
new[col + '_max'] = actual.groupby(['series_id'])[col].max()
new[col + '_std'] = actual.groupby(['series_id'])[col].std()
new[col + '_max_to_min'] = new[col + '_max'] / new[col + '_min']
new[col + '_mean_abs_change'] = actual.groupby('series_id')[col].apply(f2)
 new[col + '_mean_change_of_abs_change'] = actual.groupby('series_id')[col].apply(f1)
 new[col + '_abs_max'] = actual.groupby('series_id')[col].apply(lambda x: np.max(np.abs(x)))
 new[col + '_abs_min'] = actual.groupby('series_id')[col].apply(lambda x: np.min(np.abs(x)))
 return new
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch, model, loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )<save_to_csv>
%%time
xtrain = data_engineering(X_train)
 xtest = data_engineering(X_test )
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )<import_modules>
le = LabelEncoder()
y_train['surface'] = le.fit_transform(y_train['surface'] )
import os
from pathlib import Path
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot as plt
from sklearn.base import TransformerMixin
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split, KFold
from sklearn.ensemble import RandomForestRegressor
from sklearn.inspection import permutation_importance
from sklearn.feature_selection import chi2, SelectKBest
from sklearn.pipeline import Pipeline
from sklearn.metrics import mean_absolute_error
import optuna
from xgboost import XGBRegressor<load_from_csv>
xtrain.fillna(0, inplace = True)
 xtrain.replace(-np.inf, 0, inplace = True)
 xtrain.replace(np.inf, 0, inplace = True)
 xtest.fillna(0, inplace = True)
 xtest.replace(-np.inf, 0, inplace = True)
 xtest.replace(np.inf, 0, inplace = True )
data_folder = Path('.. /input/predict-volcanic-eruptions-ingv-oe/')
 df = pd.read_csv(data_folder / 'train.csv')
 df.head().T<load_from_csv>
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=11)
 def feed_model(train,test,label,folds=folds,averaging='usual',clf=None,clf_type='rfc',params=None,
plot_feature_importance=False,groups=y_train['group_id']):
forcast = np.zeros(( test.shape[0], 9))
con_pred = np.zeros(( train.shape[0]))
scores = []
feature_importance = pd.DataFrame()
for fold_,(train_index, valid_index)in enumerate(folds.split(train, label, groups)) :
print('Fold', fold_, 'started at', time.ctime())
 x_train, x_valid = train.iloc[train_index], train.iloc[valid_index]
y_train, y_valid = label.iloc[train_index], label.iloc[valid_index]
if clf_type == 'rfc':
clf = clf
clf.fit(x_train, y_train)
 Valid_pred = clf.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, Valid_pred)
 Real_pred = clf.predict_proba(test)
 con_pred[valid_index] = clf.predict(x_valid ).reshape(-1,)
 scores.append(accuracy_score(y_valid, Valid_pred))
if averaging == 'usual':
forcast += Real_pred
elif averaging == 'rank':
forcast += pd.Series(Real_pred ).rank().values
forcast /= folds.n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if clf_type == 'lgb':
feature_importance["importance"] /= folds.n_splits
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(by="importance",
ascending=False)[:50].index
best_features = feature_importance[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return Valid_pred, forcast, feature_importance
return Valid_pred, forcast, scores
else:
return Valid_pred, forcast, scores
df = df.sort_values('time_to_eruption')
 time_min = df.head(1 ).iloc[0]['segment_id']
time_max = df.tail(1 ).iloc[0]['segment_id']
df_min = pd.read_csv(data_folder / 'train' / f'{time_min}.csv')
 df_max = pd.read_csv(data_folder / 'train' / f'{time_max}.csv')
 df_min['time_to_eruption'] = df.head(1 ).iloc[0]['time_to_eruption']
df_max['time_to_eruption'] = df.tail(1 ).iloc[0]['time_to_eruption']<drop_column>
sfm = SelectFromModel(clf, threshold=0.001)
 sfm.fit(xtrain, y_train['surface'])
 for feature_list_index in sfm.get_support(indices=True):
print(feat_labels[feature_list_index])
 xtrain_importance = sfm.transform(xtrain)
 xtest_importance = sfm.transform(xtest )
for i in range(10):
sensor = f'sensor_{i + 1}'
if df_min[sensor].isnull().all() or df_max[sensor].isnull().all() :
del df_min[sensor]
del df_max[sensor]<import_modules>
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=11)
 def feed_model_importance(train,test,label,folds=folds,averaging='usual',clf=None,clf_type='rfc',params=None,
plot_feature_importance=False,groups=y_train['group_id']):
forcast = np.zeros(( test.shape[0], 9))
con_pred = np.zeros(( train.shape[0]))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(train, label, groups)) :
print('Fold', fold_n, 'started at', time.ctime())
 x_train, x_valid = train[train_index], train[valid_index]
y_train, y_valid = label[train_index], label[valid_index]
if clf_type == 'rfc':
clf = clf
clf.fit(x_train, y_train)
 Valid_pred = clf.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, Valid_pred)
 Real_pred = clf.predict_proba(test)
 con_pred[valid_index] +=(Valid_pred ).reshape(-1,)
 scores.append(accuracy_score(y_valid, Valid_pred))
if averaging == 'usual':
forcast += Real_pred
elif averaging == 'rank':
forcast += pd.Series(Real_pred ).rank().values
forcast /= folds.n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if clf_type == 'lgb':
feature_importance["importance"] /= n_folds
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return Valid_pred, forcast, feature_importance
return Valid_pred, forcast, scores
else:
return Valid_pred, forcast, scores
from tsfresh import extract_features
from tsfresh.feature_extraction import MinimalFCParameters<feature_engineering>
np.random.seed(0)
 folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=11)
 def create_network() :
network = models.Sequential()
network.add(layers.Dense(units=16, activation='relu', input_dim= 171))
network.add(layers.Dense(units=16, activation='relu'))
network.add(layers.Dense(units=1, activation='sigmoid'))
network.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return network
es = EarlyStopping(monitor='val_loss',mode='auto',verbose=1,patience=20)
 mc = ModelCheckpoint('best_model.h5',monitor='val_acc',mode='max',verbose=1,save_best_only=True)
 neural_network = KerasClassifier(build_fn=create_network,epochs=10,batch_size=100,verbose=0)
 cross_val_score(neural_network, xtrain, y_train['surface'],fit_params={'callbacks': [es,mc]}, cv=folds )
tsfresh_parameters = MinimalFCParameters()<init_hyperparams>
sub = pd.read_csv(os.path.join(location,'sample_submission.csv'))
sub['surface'] = le.inverse_transform(forcast_rfc.argmax(axis=1))
sub.to_csv('Forcasting.csv', index=False)
 sub.head(20 )