Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<train_on_grid>
print(pd.__version__)


 print(os.listdir(".. /input"))
logit = LogisticRegressionCV(solver='liblinear', cv=5, dual=False, penalty='l1', multi_class='ovr')
 logit.fit(train_X, train_y)
 score = logit.score(val_X, val_y)
 print("Logistic Regression:", score)
 print(logit.predict(val_X))
display([p[1] for p in logit.predict_proba(val_X)])
 clf = KNeighborsClassifier(n_neighbors=3, weights='distance', algorithm='auto', p=2)
 clf.fit(train_X, train_y)
 score = clf.score(val_X, val_y)
 print("KNN:", score)
 for k in [RBF() , Matern() , RationalQuadratic() , DotProduct() ]:
gpc = GaussianProcessClassifier(kernel=k, n_restarts_optimizer=5, max_iter_predict=200)
 gpc.fit(train_X, train_y)
 score = gpc.score(val_X, val_y)
 print("GaussianProcess", k, score)
 nb = GaussianNB()
nb.fit(train_X, train_y)
 score = nb.score(val_X, val_y)
 print("Naive Bayes:", score)
 print(nb.predict(val_X))
display([p[1] for p in nb.predict_proba(val_X)] )<train_model>
!pip install bert-tensorflow
<find_best_model_class>
!wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip
logit = LogisticRegressionCV(solver='liblinear', cv=5, dual=False, penalty='l1', multi_class='ovr')
 logit.fit(train_data_filtered, train_label_int)
 logit_prediction = [p[1] for p in logit.predict_proba(test_data_filtered)]
nb = GaussianNB()
nb.fit(train_data_filtered, train_label_int)
 nb_prediction = [p[1] for p in nb.predict_proba(test_data_filtered)]
sub_preds = [(i + j)/ 2 for i, j in zip(logit_prediction, nb_prediction)]
submission = [[a, b] for a, b in zip(test_id.values, sub_preds)]
submission = pd.DataFrame(submission, columns=['id', 'target'])
 display(submission.head())
 print("len(submission)", len(submission))
print("len(target ~ 1)", len(submission[submission.target >= 0.5]))
print("len(target ~ 0)", len(submission[submission.target < 0.5]))
sns.distplot(submission['target'], hist=True, kde=True, color = 'blue', bins = 25 ).set_title('Predictions')
 plt.show()
submission.to_csv("submission.csv", index=False )<import_modules>
folder = 'model_folder'
with zipfile.ZipFile("uncased_L-12_H-768_A-12.zip","r")as zip_ref:
zip_ref.extractall(folder )
import pandas as pd<import_modules>
import bert
from bert import run_classifier
from bert import optimization
from bert import tokenization
from bert import modeling
import pandas as pd<load_from_csv>
BERT_MODEL = 'uncased_L-12_H-768_A-12'
BERT_PRETRAINED_DIR = f'{folder}/uncased_L-12_H-768_A-12'
OUTPUT_DIR = '.. /output'
print(f'>> Model output directory: {OUTPUT_DIR}')
 print(f'>> BERT pretrained directory: {BERT_PRETRAINED_DIR}')
 BERT_VOCAB= '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/vocab.txt'
BERT_INIT_CHKPNT = '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/bert_model.ckpt'
BERT_CONFIG = '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/bert_config.json'
df1 = pd.read_csv('.. /input/lanl-blend-csv/blend_ver1.csv')
 df2 = pd.read_csv('.. /input/lanl-blend-csv/blend_ver2.csv')
 df3 = pd.read_csv('.. /input/lanl-blend-csv/blend_ver3.csv' )<load_from_csv>
tokenization.validate_case_matches_checkpoint(True,BERT_INIT_CHKPNT)
 tokenizer = tokenization.FullTokenizer(
 vocab_file=BERT_VOCAB, do_lower_case=True )
blend = df1['time_to_failure'] *0.33333333334 + df2['time_to_failure'] * 0.33333333333 + df3['time_to_failure'] * 0.33333333333
sample = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv' )<save_to_csv>
%matplotlib inline
warnings.filterwarnings("ignore")
 le = LabelEncoder()
df = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 df.drop('keyword', axis=1)
 df.drop('location', axis=1)
 df.head()
sample['time_to_failure'] = blend
sample.to_csv('blend_ver4.csv',index=False )<define_variables>
df['text'] = df['text'].apply(lambda x : re.sub(r'\b[A-Z]+\b', '',re.sub('\s+',' ',re.sub(r'[^a-zA-Z]', ' ',str(x)))))
 df['text'] = df['text'].apply(lambda x : x.lower())
 identity_columns = [0,1]
df2 = pd.DataFrame({'text':df['text'].replace(r'
',' ',regex=True),
'label':df['target'].replace(r' ','',regex=True),
})
 X_train, X_test, y_train, y_test = train_test_split(df2["text"].values, df2["label"].values, test_size=0.2, random_state=42)
 X_train, y_train = df2["text"].values, df2["label"].values
testpd = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 testpd['text'] = testpd['text'].apply(lambda x : re.sub(r'\b[A-Z]+\b', '',re.sub('\s+',' ',re.sub(r'[^a-zA-Z]', ' ',str(x)))))
 testpd['text'] = testpd['text'].apply(lambda x : x.lower())
 test = pd.DataFrame({'text':testpd['text'].replace(r'
',' ',regex=True)})
 test = test["text"].values
df4 = sample<load_from_csv>
def create_examples(lines, set_type, labels=None):
guid=f'{set_type}'
examples = []
if set_type == 'train':
for line, label in zip(lines, labels):
text_a = line
label = str(label)
 examples.append(
 run_classifier.InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
else:
for line in lines:
text_a = line
label = '0'
examples.append(
 run_classifier.InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
return examples
TRAIN_BATCH_SIZE = 32
EVAL_BATCH_SIZE = 8
LEARNING_RATE = 1e-5
NUM_TRAIN_EPOCHS = 3.0
WARMUP_PROPORTION = 0.1
MAX_SEQ_LENGTH = 100
SAVE_CHECKPOINTS_STEPS = 1000
ITERATIONS_PER_LOOP = 100
NUM_TPU_CORES = 8
VOCAB_FILE = os.path.join(BERT_PRETRAINED_DIR, 'vocab.txt')
 CONFIG_FILE = os.path.join(BERT_PRETRAINED_DIR, 'bert_config.json')
 INIT_CHECKPOINT = os.path.join(BERT_PRETRAINED_DIR, 'bert_model.ckpt')
 DO_LOWER_CASE = BERT_MODEL.startswith('uncased')
 label_list = [str(num)for num in range(38)]
tokenizer = tokenization.FullTokenizer(vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE)
 train_examples = create_examples(X_train, 'train', labels=y_train)
 tpu_cluster_resolver = None
run_config = tf.contrib.tpu.RunConfig(
 cluster=tpu_cluster_resolver,
model_dir=OUTPUT_DIR,
save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS,
tpu_config=tf.contrib.tpu.TPUConfig(
 iterations_per_loop=ITERATIONS_PER_LOOP,
num_shards=NUM_TPU_CORES,
per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2))
num_train_steps = int(
 len(train_examples)/ TRAIN_BATCH_SIZE * NUM_TRAIN_EPOCHS)
 num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)
 model_fn = run_classifier.model_fn_builder(
 bert_config=modeling.BertConfig.from_json_file(CONFIG_FILE),
num_labels=len(label_list),
init_checkpoint=INIT_CHECKPOINT,
learning_rate=LEARNING_RATE,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
use_tpu=False,
use_one_hot_embeddings=True)
 estimator = tf.contrib.tpu.TPUEstimator(
 use_tpu=False,
model_fn=model_fn,
config=run_config,
train_batch_size=TRAIN_BATCH_SIZE,
eval_batch_size=EVAL_BATCH_SIZE )
blend1 = df1['time_to_failure'] *0.33333333333 + df2['time_to_failure'] * 0.33333333333 + df4['time_to_failure'] * 0.33333333334
sample1 = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv' )<save_to_csv>
print('Please wait...')
 train_features = run_classifier.convert_examples_to_features(
 train_examples, label_list, MAX_SEQ_LENGTH, tokenizer)
 print('>> Started training at {} '.format(datetime.now()))
print(' Num examples = {}'.format(len(train_examples)))
 print(' Batch size = {}'.format(TRAIN_BATCH_SIZE))
tf.logging.info(" Num steps = %d", num_train_steps)
 train_input_fn = run_classifier.input_fn_builder(
 features=train_features,
seq_length=MAX_SEQ_LENGTH,
is_training=True,
drop_remainder=True)
 estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
 print('>> Finished training at {}'.format(datetime.now()))
sample1['time_to_failure'] = blend1
sample1.to_csv('blend_ver5.csv',index=False )<define_variables>
def input_fn_builder(features, seq_length, is_training, drop_remainder):

 all_input_ids = []
all_input_mask = []
all_segment_ids = []
all_label_ids = []
for feature in features:
all_input_ids.append(feature.input_ids)
 all_input_mask.append(feature.input_mask)
 all_segment_ids.append(feature.segment_ids)
 all_label_ids.append(feature.label_id)
 def input_fn(params):

 print(params)
 batch_size = 500
num_examples = len(features)
 d = tf.data.Dataset.from_tensor_slices({
"input_ids":
tf.constant(
 all_input_ids, shape=[num_examples, seq_length],
dtype=tf.int32),
"input_mask":
tf.constant(
 all_input_mask,
shape=[num_examples, seq_length],
dtype=tf.int32),
"segment_ids":
tf.constant(
 all_segment_ids,
shape=[num_examples, seq_length],
dtype=tf.int32),
"label_ids":
tf.constant(all_label_ids, shape=[num_examples], dtype=tf.int32),
})
 if is_training:
d = d.repeat()
d = d.shuffle(buffer_size=100)
 d = d.batch(batch_size=batch_size, drop_remainder=drop_remainder)
 return d
return input_fn
df5 = sample1<load_from_csv>
predict_examples = create_examples(X_test, 'test')
 predict_features = run_classifier.convert_examples_to_features(
 predict_examples, label_list, MAX_SEQ_LENGTH, tokenizer)
 predict_input_fn = input_fn_builder(
 features=predict_features,
seq_length=MAX_SEQ_LENGTH,
is_training=False,
drop_remainder=False)
 result = estimator.predict(input_fn=predict_input_fn )
blend2 = df1['time_to_failure'] *0.2 + df2['time_to_failure'] * 0.2 + df3['time_to_failure'] * 0.2 + df4['time_to_failure'] * 0.2 + df5['time_to_failure'] * 0.2
sample2 = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv' )<save_to_csv>
preds = []
for prediction in result:
preds.append(np.argmax(prediction['probabilities']))
sample2['time_to_failure'] = blend2
sample2.to_csv('blend_ver6.csv',index=False )<define_variables>
from sklearn.metrics import accuracy_score
df6 = sample2<load_from_csv>
print("Accuracy of BERT is:",accuracy_score(y_test,preds))
blend3 = df1['time_to_failure'] *0.16666666666 + df2['time_to_failure'] * 0.16666666666 + df3['time_to_failure'] * 0.16666666666 + df4['time_to_failure'] * 0.16666666666 + df5['time_to_failure'] * 0.16666666666 + df6['time_to_failure'] * 0.16666666666
sample3 = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv' )<save_to_csv>
print(classification_report(y_test,preds))
sample3['time_to_failure'] = blend3
sample3.to_csv('blend_ver7.csv',index=False )<set_options>
testpd = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 testpd['text'] = testpd['text'].apply(lambda x : re.sub(r'\b[A-Z]+\b', '',re.sub('\s+',' ',re.sub(r'[^a-zA-Z]', ' ',str(x)))))
 testpd['text'] = testpd['text'].apply(lambda x : x.lower())
 testln = pd.DataFrame({'id':testpd['id'],
'text':testpd['text'].replace(r'
',' ',regex=True)})
 testl = testln["text"].values
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.simplefilter(action='ignore', category=UserWarning)
 warnings.simplefilter(action='ignore', category=RuntimeWarning)
 init_notebook_mode(connected=True)
 RANDOM_SEED = 19
np.random.seed(RANDOM_SEED)
 def lineplot(x_axis, y_axis, title=''):
trace = go.Scatter(x=x_axis, y=y_axis, mode='lines+markers')
 layout = go.Layout(
 title=title,
showlegend=False,
xaxis=dict(
 title='Number of features',
titlefont=dict(size=14, color=' 
),
yaxis=dict(
 title='Validation score',
titlefont=dict(size=14, color=' 
)
)
 fig = go.Figure(data=[trace], layout=layout)
 iplot(fig )<load_from_csv>
predict_test = create_examples(testl, 'test')
 predict_features1 = run_classifier.convert_examples_to_features(
 predict_test, label_list, MAX_SEQ_LENGTH, tokenizer)
 predict_input_fn1 = input_fn_builder(
 features=predict_features1,
seq_length=MAX_SEQ_LENGTH,
is_training=False,
drop_remainder=False)
 result1 = estimator.predict(input_fn=predict_input_fn1 )
data_type = {'acoustic_data': np.int16, 'time_to_failure': np.float64}
train = pd.read_csv('.. /input/train.csv', dtype=data_type)
 train.head(3 )<statistical_test>
preds1 = []
for prediction in result1 :
preds1.append(np.argmax(prediction['probabilities']))
rows = 150_000
segments = int(np.floor(train.shape[0] / rows))
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta
X_tr = pd.DataFrame(index=range(segments), dtype=np.float64)
 y_tr = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])
 total_mean = train['acoustic_data'].mean()
total_std = train['acoustic_data'].std()
total_max = train['acoustic_data'].max()
total_min = train['acoustic_data'].min()
total_sum = train['acoustic_data'].sum()
total_abs_sum = np.abs(train['acoustic_data'] ).sum()
def calc_change_rate(x):
change =(np.diff(x)/ x[:-1] ).values
change = change[np.nonzero(change)[0]]
change = change[~np.isnan(change)]
change = change[change != -np.inf]
change = change[change != np.inf]
return np.mean(change)
 for segment in range(segments):
seg = train.iloc[segment*rows:segment*rows+rows]
x = pd.Series(seg['acoustic_data'].values)
 y = seg['time_to_failure'].values[-1]
y_tr.loc[segment, 'time_to_failure'] = y
X_tr.loc[segment, 'mean'] = x.mean()
X_tr.loc[segment, 'std'] = x.std()
X_tr.loc[segment, 'max'] = x.max()
X_tr.loc[segment, 'min'] = x.min()
X_tr.loc[segment, 'mean_change_abs'] = np.mean(np.diff(x))
X_tr.loc[segment, 'mean_change_rate'] = calc_change_rate(x)
 X_tr.loc[segment, 'abs_max'] = np.abs(x ).max()
X_tr.loc[segment, 'std_first_50000'] = x[:50000].std()
X_tr.loc[segment, 'std_last_50000'] = x[-50000:].std()
X_tr.loc[segment, 'std_first_10000'] = x[:10000].std()
X_tr.loc[segment, 'std_last_10000'] = x[-10000:].std()
X_tr.loc[segment, 'avg_first_50000'] = x[:50000].mean()
X_tr.loc[segment, 'avg_last_50000'] = x[-50000:].mean()
X_tr.loc[segment, 'avg_first_10000'] = x[:10000].mean()
X_tr.loc[segment, 'avg_last_10000'] = x[-10000:].mean()
X_tr.loc[segment, 'min_first_50000'] = x[:50000].min()
X_tr.loc[segment, 'min_last_50000'] = x[-50000:].min()
X_tr.loc[segment, 'min_first_10000'] = x[:10000].min()
X_tr.loc[segment, 'min_last_10000'] = x[-10000:].min()
X_tr.loc[segment, 'max_first_50000'] = x[:50000].max()
X_tr.loc[segment, 'max_last_50000'] = x[-50000:].max()
X_tr.loc[segment, 'max_first_10000'] = x[:10000].max()
X_tr.loc[segment, 'max_last_10000'] = x[-10000:].max()
X_tr.loc[segment, 'max_to_min'] = x.max() / np.abs(x.min())
 X_tr.loc[segment, 'max_to_min_diff'] = x.max() - np.abs(x.min())
 X_tr.loc[segment, 'count_big'] = len(x[np.abs(x)> 500])
 X_tr.loc[segment, 'sum'] = x.sum()
X_tr.loc[segment, 'mean_change_rate_first_50000'] = calc_change_rate(x[:50000])
 X_tr.loc[segment, 'mean_change_rate_last_50000'] = calc_change_rate(x[-50000:])
 X_tr.loc[segment, 'mean_change_rate_first_10000'] = calc_change_rate(x[:10000])
 X_tr.loc[segment, 'mean_change_rate_last_10000'] = calc_change_rate(x[-10000:])
 X_tr.loc[segment, 'q95'] = np.quantile(x, 0.95)
 X_tr.loc[segment, 'q99'] = np.quantile(x, 0.99)
 X_tr.loc[segment, 'q05'] = np.quantile(x, 0.05)
 X_tr.loc[segment, 'q01'] = np.quantile(x, 0.01)
 X_tr.loc[segment, 'abs_q95'] = np.quantile(np.abs(x), 0.95)
 X_tr.loc[segment, 'abs_q99'] = np.quantile(np.abs(x), 0.99)
 X_tr.loc[segment, 'abs_q05'] = np.quantile(np.abs(x), 0.05)
 X_tr.loc[segment, 'abs_q01'] = np.quantile(np.abs(x), 0.01)
 X_tr.loc[segment, 'trend'] = add_trend_feature(x)
 X_tr.loc[segment, 'abs_trend'] = add_trend_feature(x, abs_values=True)
 X_tr.loc[segment, 'abs_mean'] = np.abs(x ).mean()
X_tr.loc[segment, 'abs_std'] = np.abs(x ).std()
X_tr.loc[segment, 'mad'] = x.mad()
X_tr.loc[segment, 'kurt'] = x.kurtosis()
X_tr.loc[segment, 'skew'] = x.skew()
X_tr.loc[segment, 'med'] = x.median()
X_tr.loc[segment, 'Hilbert_mean'] = np.abs(hilbert(x)).mean()
X_tr.loc[segment, 'Hann_window_mean'] =(convolve(x, hann(150), mode='same')/ sum(hann(150)) ).mean()
X_tr.loc[segment, 'classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000 ).mean()
X_tr.loc[segment, 'classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000 ).mean()
X_tr.loc[segment, 'classic_sta_lta3_mean'] = classic_sta_lta(x, 3333, 6666 ).mean()
X_tr.loc[segment, 'classic_sta_lta4_mean'] = classic_sta_lta(x, 10000, 25000 ).mean()
X_tr.loc[segment, 'classic_sta_lta5_mean'] = classic_sta_lta(x, 50, 1000 ).mean()
X_tr.loc[segment, 'classic_sta_lta6_mean'] = classic_sta_lta(x, 100, 5000 ).mean()
X_tr.loc[segment, 'classic_sta_lta7_mean'] = classic_sta_lta(x, 333, 666 ).mean()
X_tr.loc[segment, 'classic_sta_lta8_mean'] = classic_sta_lta(x, 4000, 10000 ).mean()
X_tr.loc[segment, 'Moving_average_700_mean'] = x.rolling(window=700 ).mean().mean(skipna=True)
 ewma = pd.Series.ewm
X_tr.loc[segment, 'exp_Moving_average_300_mean'] =(ewma(x, span=300 ).mean() ).mean(skipna=True)
 X_tr.loc[segment, 'exp_Moving_average_3000_mean'] = ewma(x, span=3000 ).mean().mean(skipna=True)
 X_tr.loc[segment, 'exp_Moving_average_30000_mean'] = ewma(x, span=30000 ).mean().mean(skipna=True)
 no_of_std = 3
X_tr.loc[segment, 'MA_700MA_std_mean'] = x.rolling(window=700 ).std().mean()
X_tr.loc[segment,'MA_700MA_BB_high_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] + no_of_std * X_tr.loc[segment, 'MA_700MA_std_mean'] ).mean()
X_tr.loc[segment,'MA_700MA_BB_low_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] - no_of_std * X_tr.loc[segment, 'MA_700MA_std_mean'] ).mean()
X_tr.loc[segment, 'MA_400MA_std_mean'] = x.rolling(window=400 ).std().mean()
X_tr.loc[segment,'MA_400MA_BB_high_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] + no_of_std * X_tr.loc[segment, 'MA_400MA_std_mean'] ).mean()
X_tr.loc[segment,'MA_400MA_BB_low_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] - no_of_std * X_tr.loc[segment, 'MA_400MA_std_mean'] ).mean()
X_tr.loc[segment, 'MA_1000MA_std_mean'] = x.rolling(window=1000 ).std().mean()
X_tr.drop('Moving_average_700_mean', axis=1, inplace=True)
 X_tr.loc[segment, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))
X_tr.loc[segment, 'q999'] = np.quantile(x,0.999)
 X_tr.loc[segment, 'q001'] = np.quantile(x,0.001)
 X_tr.loc[segment, 'ave10'] = stats.trim_mean(x, 0.1)
 for windows in [10, 100, 1000]:
x_roll_std = x.rolling(windows ).std().dropna().values
x_roll_mean = x.rolling(windows ).mean().dropna().values
X_tr.loc[segment, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X_tr.loc[segment, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X_tr.loc[segment, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X_tr.loc[segment, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X_tr.loc[segment, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X_tr.loc[segment, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X_tr.loc[segment, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X_tr.loc[segment, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X_tr.loc[segment, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X_tr.loc[segment, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_tr.loc[segment, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X_tr.loc[segment, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X_tr.loc[segment, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X_tr.loc[segment, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X_tr.loc[segment, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X_tr.loc[segment, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X_tr.loc[segment, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X_tr.loc[segment, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X_tr.loc[segment, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X_tr.loc[segment, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_tr.loc[segment, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_tr.loc[segment, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()
means_dict = {}
for col in X_tr.columns:
if X_tr[col].isnull().any() :
mean_value = X_tr.loc[X_tr[col] != -np.inf, col].mean()
X_tr.loc[X_tr[col] == -np.inf, col] = mean_value
X_tr[col] = X_tr[col].fillna(mean_value)
 means_dict[col] = mean_value
print("Original shape: {}, final shape: {}".format(train.shape, X_tr.shape))
X_tr.head(3 )<load_from_csv>
def create_output(predictions):
probabilities = []
for(i, prediction)in enumerate(predictions):
preds = prediction
probabilities.append(preds)
 dff = pd.DataFrame(probabilities)
 dff.head()
return dff
<train_model><EOS> 
predslab=[identity_columns[x] for x in preds1]
testln["target"]=predslab
merged_df = testln
submission = merged_df.drop(['text'], axis=1)
 submission.to_csv("submission.csv", index=False )