Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<define_variables>
%matplotlib inline
warnings.filterwarnings('ignore' )
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']<data_type_conversions>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
max_features = None
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))<string_transform>
X_train = train_df.drop(["ID", "target"], axis=1)
 y_train = np.log1p(train_df["target"].values)
 X_test = test_df.drop(["ID"], axis=1 )
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<concatenate>
colsToRemove = []
for col in X_train.columns:
if X_train[col].std() == 0:
colsToRemove.append(col)
 X_train.drop(colsToRemove, axis=1, inplace=True)
 X_test.drop(colsToRemove, axis=1, inplace=True)
 print("Removed `{}` Constant Columns
".format(len(colsToRemove)))
 print(colsToRemove )
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()<prepare_x_and_y>
%%time
colsToRemove = []
colsScaned = []
dupList = {}
columns = X_train.columns
for i in range(len(columns)-1):
v = X_train[columns[i]].values
dupCols = []
for j in range(i+1,len(columns)) :
if np.array_equal(v, X_train[columns[j]].values):
colsToRemove.append(columns[j])
 if columns[j] not in colsScaned:
dupCols.append(columns[j])
 colsScaned.append(columns[j])
 dupList[columns[i]] = dupCols
X_train.drop(colsToRemove, axis=1, inplace=True)
 X_test.drop(colsToRemove, axis=1, inplace=True)
 print("Removed `{}` Duplicate Columns
".format(len(dupList)))
 print(dupList )
x_train_torch = torch.tensor(x_train, dtype=torch.long)
 y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32)
 x_test_torch = torch.tensor(x_test, dtype=torch.long)
 batch_size = 512
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 valid_dataset = data.TensorDataset(x_train_torch[:batch_size], y_train_torch[:batch_size])
 test_dataset = data.TensorDataset(x_test_torch)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
 databunch = DataBunch(train_dl=train_loader,valid_dl=valid_loader )<choose_model_class>
def drop_sparse(train, test):
flist = [x for x in train.columns if not x in ['ID','target']]
for f in flist:
if len(np.unique(train[f])) <2:
train.drop(f, axis=1, inplace=True)
 test.drop(f, axis=1, inplace=True)
 return train, test
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch,model,loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )<save_to_csv>
%%time
X_train, X_test = drop_sparse(X_train, X_test )
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': 0.88*test_pred + 0.12*(test_prednp.mean(all_test_preds, axis=0)[:, 0])})
 submission.to_csv('submission.csv', index=False )<save_to_csv>
def add_SumZeros(train, test, features):
flist = [x for x in train.columns if not x in ['ID','target']]
if 'SumZeros' in features:
train.insert(1, 'SumZeros',(train[flist] == 0 ).astype(int ).sum(axis=1))
test.insert(1, 'SumZeros',(test[flist] == 0 ).astype(int ).sum(axis=1))
flist = [x for x in train.columns if not x in ['ID','target']]
return train, test
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': 0.78*test_pred + 0.22*(test_prednp.mean(all_test_preds, axis=0)[:, 0])})
 submission.to_csv('submission_v.csv', index=False )<save_to_csv>
%%time
X_train, X_test = add_SumZeros(X_train, X_test, ['SumZeros'] )
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': 0.68*test_pred + 0.32*(test_prednp.mean(all_test_preds, axis=0)[:, 0])})
 submission.to_csv('submission_v1.csv', index=False )<define_search_model>
def add_SumValues(train, test, features):
flist = [x for x in train.columns if not x in ['ID','target']]
if 'SumValues' in features:
train.insert(1, 'SumValues',(train[flist] != 0 ).astype(int ).sum(axis=1))
test.insert(1, 'SumValues',(test[flist] != 0 ).astype(int ).sum(axis=1))
flist = [x for x in train.columns if not x in ['ID','target']]
return train, test
ps = PorterStemmer()
ls = LancasterStemmer()
ss = SnowballStemmer('english' )<load_from_csv>
%%time
X_train, X_test = add_SumValues(X_train, X_test, ['SumValues'] )
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<feature_engineering>
def add_OtherAgg(train, test, features):
flist = [x for x in train.columns if not x in ['ID','target','SumZeros','SumValues']]
if 'OtherAgg' in features:
train['Mean'] = train[flist].mean(axis=1)
 train['Median'] = train[flist].median(axis=1)
 train['Mode'] = train[flist].mode(axis=1)
 train['Max'] = train[flist].max(axis=1)
 train['Var'] = train[flist].var(axis=1)
 train['Std'] = train[flist].std(axis=1)
 test['Mean'] = test[flist].mean(axis=1)
 test['Median'] = test[flist].median(axis=1)
 test['Mode'] = test[flist].mode(axis=1)
 test['Max'] = test[flist].max(axis=1)
 test['Var'] = test[flist].var(axis=1)
 test['Std'] = test[flist].std(axis=1)
 flist = [x for x in train.columns if not x in ['ID','target','SumZeros','SumValues']]
return train, test
identity_columns = ['male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish', 'muslim', 'black', 'white',
'psychiatric_or_mental_illness']
for col in identity_columns+['target']:
train[col] = np.where(train[col]>=0.5, True, False )<set_options>
%%time
X_train, X_test = add_OtherAgg(X_train, X_test, ['OtherAgg'] )
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
set_random_seed(2)
 seed_everything()<define_variables>
from sklearn.random_projection import GaussianRandomProjection, SparseRandomProjection
from sklearn.decomposition import TruncatedSVD, FastICA
max_features = 150000
maxlen = 256
embed_size = 302<define_variables>
PERC_TRESHOLD = 0.98
N_COMP = 97
print("tSVD")
 tsvd = TruncatedSVD(n_components=N_COMP, random_state=17)
 tsvd_results_train = tsvd.fit_transform(X_train)
 tsvd_results_test = tsvd.transform(X_test)
 print("Append decomposition components to datasets...")
 for i in range(1, N_COMP + 1):
X_train['tsvd_' + str(i)] = tsvd_results_train[:, i - 1]
X_test['tsvd_' + str(i)] = tsvd_results_test[:, i - 1]
print('
Train shape: {}
Test shape: {}'.format(X_train.shape, X_test.shape))
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]<drop_column>
import lightgbm as lgb
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x<categorify>
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
'boosting_type' : 'goss',
'max_depth' : 5,
"num_leaves" : 20,
"learning_rate" : 0.01,
"feature_fraction" : 0.6,
"bagging_seed" : 42,
"verbosity" : -1,
'lambda_l2' : 0.000001,
'lambda_l1' : 0.00001,
'max_bin' : 200
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 1000, valid_sets=[lgtrain, lgval], early_stopping_rounds=100,
verbose_eval=200, evals_result=evals_result)
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 return pred_test_y, model, evals_result
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x 
<define_variables>
seeds = [42]
pred_test_full_seed = 0
for seed in seeds:
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=seed)
 pred_test_full = 0
for dev_index, val_index in kf.split(X_train):
dev_X, val_X = X_train.loc[dev_index,:], X_train.loc[val_index,:]
dev_y, val_y = y_train[dev_index], y_train[val_index]
pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, X_test)
 pred_test_full += pred_test
pred_test_full /= 5.
 pred_test_full = np.expm1(pred_test_full)
 pred_test_full_seed += pred_test_full
print("Seed {} completed.... ".format(seed))
pred_test_full_seed /= np.float(len(seeds))
print("LightGBM Training Completed..." )
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }<string_transform>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub_lgb = pd.DataFrame()
sub_lgb["target"] = pred_test_full_seed
sub["target"] = sub_lgb["target"]
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text<split>
print(sub.head())
 sub.to_csv('Ensemble9.csv', index=False )