Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<train_model>
plt.rcParams.update({'font.size': 14})
 train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 sub_sample = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 print(train.shape, test.shape, sub_sample.shape )
%%time
nfolds=5
aml = H2OAutoML(max_models=500, seed=554, max_runtime_secs=8000,nfolds=nfolds,sort_metric="MAE")
 aml.train(x=x, y=y, training_frame=train )<set_options>
train.duplicated().sum()
aml.leader<save_to_csv>
train = train.drop_duplicates().reset_index(drop=True )
preds = aml.predict(test)
 sample_submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sample_submission['time_to_failure'] = preds.as_data_frame().values.flatten()
sample_submission.to_csv('h2o_submission_fold5.csv', index=False )<save_to_csv>
train.isnull().sum()
sample_submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sample_submission['time_to_failure'] =(pd.read_csv('h2o_submission_fold0.csv')['time_to_failure']+pd.read_csv('h2o_submission_fold5.csv')['time_to_failure'])/2
sample_submission.to_csv('h2o_submission_blend.csv', index=False )<set_options>
test.isnull().sum()
warnings.filterwarnings("ignore" )<install_modules>
test.isnull().sum()
!pip install tsfresh<import_modules>
print(train.keyword.nunique() , test.keyword.nunique())
 print(set(train.keyword.unique())- set(test.keyword.unique()))
from tsfresh.feature_extraction import feature_calculators<install_modules>
print(train.location.nunique() , test.location.nunique() )
!pip install tpot 
<load_from_csv>
for col in ['keyword','location']:
train[col] = train[col].fillna('None')
 test[col] = test[col].fillna('None')
 def clean_loc(x):
if x == 'None':
return 'None'
elif x == 'Earth' or x =='Worldwide' or x == 'Everywhere':
return 'World'
elif 'New York' in x or 'NYC' in x:
return 'New York'
elif 'London' in x:
return 'London'
elif 'Mumbai' in x:
return 'Mumbai'
elif 'Washington' in x and 'D' in x and 'C' in x:
return 'Washington DC'
elif 'San Francisco' in x:
return 'San Francisco'
elif 'Los Angeles' in x:
return 'Los Angeles'
elif 'Seattle' in x:
return 'Seattle'
elif 'Chicago' in x:
return 'Chicago'
elif 'Toronto' in x:
return 'Toronto'
elif 'Sacramento' in x:
return 'Sacramento'
elif 'Atlanta' in x:
return 'Atlanta'
elif 'California' in x:
return 'California'
elif 'Florida' in x:
return 'Florida'
elif 'Texas' in x:
return 'Texas'
elif 'United States' in x or 'USA' in x:
return 'USA'
elif 'United Kingdom' in x or 'UK' in x or 'Britain' in x:
return 'UK'
elif 'Canada' in x:
return 'Canada'
elif 'India' in x:
return 'India'
elif 'Kenya' in x:
return 'Kenya'
elif 'Nigeria' in x:
return 'Nigeria'
elif 'Australia' in x:
return 'Australia'
elif 'Indonesia' in x:
return 'Indonesia'
elif x in top_loc:
return x
else: return 'Others'
train['location_clean'] = train['location'].apply(lambda x: clean_loc(str(x)))
 test['location_clean'] = test['location'].apply(lambda x: clean_loc(str(x)) )
PATH=".. /input/"
train_df = pd.read_csv(PATH + 'train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})
 train_df.head()<define_variables>
leak = pd.read_csv(".. /input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv", encoding='latin_1')
 leak['target'] =(leak['choose_one']=='Relevant' ).astype(int)
 leak['id'] = leak.index
leak = leak[['id', 'target','text']]
merged_df = pd.merge(test, leak, on='id')
 sub1 = merged_df[['id', 'target']]
sub1.to_csv('submit_1.csv', index=False )
rows = 150000
segments = int(np.floor(train_df.shape[0] / rows))
print("Number of segments: ", segments )<prepare_x_and_y>
test_str = train.loc[417, 'text']
def clean_text(text):
text = re.sub(r'https?://\S+', '', text)
 text = re.sub(r'
',' ', text)
 text = re.sub('\s+', ' ', text ).strip()
return text
print("Original text: " + test_str)
 print("Cleaned text: " + clean_text(test_str))
train_X = pd.DataFrame(index=range(segments), dtype=np.float64)
 train_y = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'] )<normalization>
def find_hashtags(tweet):
return " ".join([match.group(0)[1:] for match in re.finditer(r"
def find_mentions(tweet):
return " ".join([match.group(0)[1:] for match in re.finditer(r"@\w+", tweet)])or 'no'
def find_links(tweet):
return " ".join([match.group(0)[:] for match in re.finditer(r"https?://\S+", tweet)])or 'no'
def process_text(df):
df['text_clean'] = df['text'].apply(lambda x: clean_text(x))
df['hashtags'] = df['text'].apply(lambda x: find_hashtags(x))
df['mentions'] = df['text'].apply(lambda x: find_mentions(x))
df['links'] = df['text'].apply(lambda x: find_links(x))
return df
train = process_text(train)
 test = process_text(test )
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta<feature_engineering>
def create_stat(df):
df['text_len'] = df['text_clean'].apply(len)
 df['word_count'] = df["text_clean"].apply(lambda x: len(str(x ).split()))
df['stop_word_count'] = df['text_clean'].apply(lambda x: len([w for w in str(x ).lower().split() if w in STOPWORDS]))
df['punctuation_count'] = df['text_clean'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
df['hashtag_count'] = df['hashtags'].apply(lambda x: len(str(x ).split()))
df['mention_count'] = df['mentions'].apply(lambda x: len(str(x ).split()))
df['link_count'] = df['links'].apply(lambda x: len(str(x ).split()))
df['caps_count'] = df['text_clean'].apply(lambda x: sum(1 for c in str(x)if c.isupper()))
df['caps_ratio'] = df['caps_count'] / df['text_len']
return df
train = create_stat(train)
 test = create_stat(test)
 print(train.shape, test.shape )
def create_features(seg_id, xc, X):
zc = np.fft.fft(xc)
 X.loc[seg_id,'cid_ce1']=feature_calculators.cid_ce(xc, 1)
 realFFT = np.real(zc)
 imagFFT = np.imag(zc)
 X.loc[seg_id, 'Rmin'] = realFFT.min()
X.loc[seg_id, 'Imin'] = imagFFT.min()
X.loc[seg_id, 'Rmax_last_15000'] = realFFT[-15000:].max()
X.loc[seg_id, 'Rmin_last_15000'] = realFFT[-15000:].min()
X.loc[seg_id, 'autocorrelation_10'] = feature_calculators.autocorrelation(xc, 10)
 X.loc[seg_id, 'Hilbert_mean'] = np.abs(hilbert(xc)).mean()
X.loc[seg_id, 'classic_sta_lta1_mean'] = classic_sta_lta(xc, 500, 10000 ).mean()
no_of_std = 2
X.loc[seg_id, 'Moving_average_700_mean'] = xc.rolling(window=700 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_400_mean'] = xc.rolling(window=400 ).mean().mean(skipna=True)
 X.loc[seg_id, 'MA_700MA_std_mean'] = xc.rolling(window=700 ).std().mean()
X.loc[seg_id,'MA_700MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id,'MA_700MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_400MA_std_mean'] = xc.rolling(window=400 ).std().mean()
X.loc[seg_id,'MA_400MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_400_mean'] + no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id,'MA_400MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_400_mean'] - no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_1000MA_std_mean'] = xc.rolling(window=1000 ).std().mean()
X.loc[seg_id,'kstat2']=sp.stats.kstat(xc, 2)
 X.loc[seg_id,'kstat3']=sp.stats.kstat(xc, 3)
 X.loc[seg_id,'abs_sum_changes']=feature_calculators.absolute_sum_of_changes(xc)
 X.loc[seg_id,'mean_abs_change']=feature_calculators.mean_abs_change(xc)
 X.loc[seg_id,'ratio_value_number_to_timeseries']=feature_calculators.ratio_value_number_to_time_series_length(xc)
 X.loc[seg_id,'ac10']=feature_calculators.autocorrelation(xc, 10)
 X.loc[seg_id,'ac50']=feature_calculators.autocorrelation(xc, 50)
 X.loc[seg_id,'npeaks_0']=feature_calculators.number_crossing_m(xc, 0)
 X.loc[seg_id,'npeaks_10']=feature_calculators.number_peaks(xc, 10)
 X.loc[seg_id,'npeaks_50']=feature_calculators.number_peaks(xc, 50)
 X.loc[seg_id,'npeaks_100']=feature_calculators.number_peaks(xc, 100)
 X.loc[seg_id,'lsbm']=feature_calculators.longest_strike_below_mean(xc)
 X.loc[seg_id,'lsam']=feature_calculators.longest_strike_above_mean(xc)
 X.loc[seg_id, 'MA_15000MA_std_mean'] = xc.rolling(window=15000 ).std().mean()
for windows in [10]:
x_roll_std = xc.rolling(windows ).std().dropna().values
x_roll_mean = xc.rolling(windows ).mean().dropna().values
X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 for windows in [100]:
X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 for windows in [1000]:
X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
<feature_engineering>
train.corr() ['target'].drop('target' ).sort_values()
<feature_engineering>
stopwords = set(STOPWORDS)
for seg_id in tqdm_notebook(range(segments)) :
seg = train_df.iloc[seg_id*rows:seg_id*rows + rows]
create_features(seg_id, seg['acoustic_data'], train_X)
 train_y.loc[seg_id, 'time_to_failure'] = seg['time_to_failure'].values[-1]<normalization>
features = ['keyword', 'location_clean']
encoder = ce.TargetEncoder(cols=features)
 encoder.fit(train[features],train['target'])
 train = train.join(encoder.transform(train[features] ).add_suffix('_target'))
test = test.join(encoder.transform(test[features] ).add_suffix('_target'))
scaler = StandardScaler()
scaler.fit(train_X)
 scaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns )<import_modules>
vec_links = CountVectorizer(min_df = 5, analyzer = 'word', token_pattern = r'https?://\S+')
 link_vec = vec_links.fit_transform(train['links'])
 link_vec_test = vec_links.transform(test['links'])
 X_train_link = pd.DataFrame(link_vec.toarray() , columns=vec_links.get_feature_names())
 X_test_link = pd.DataFrame(link_vec_test.toarray() , columns=vec_links.get_feature_names())
 vec_men = CountVectorizer(min_df = 5)
 men_vec = vec_men.fit_transform(train['mentions'])
 men_vec_test = vec_men.transform(test['mentions'])
 X_train_men = pd.DataFrame(men_vec.toarray() , columns=vec_men.get_feature_names())
 X_test_men = pd.DataFrame(men_vec_test.toarray() , columns=vec_men.get_feature_names())
 vec_hash = CountVectorizer(min_df = 5)
 hash_vec = vec_hash.fit_transform(train['hashtags'])
 hash_vec_test = vec_hash.transform(test['hashtags'])
 X_train_hash = pd.DataFrame(hash_vec.toarray() , columns=vec_hash.get_feature_names())
 X_test_hash = pd.DataFrame(hash_vec_test.toarray() , columns=vec_hash.get_feature_names())
 print(X_train_link.shape, X_train_men.shape, X_train_hash.shape )
import statsmodels.api as sm
from scipy.stats.mstats import zscore<drop_column>
hash_rank =(X_train_hash.transpose().dot(train['target'])/ X_train_hash.sum(axis=0)).sort_values(ascending=False)
 print('Hashtags with which 100% of Tweets are disasters: ')
 print(list(hash_rank[hash_rank==1].index))
print('Total: ' + str(len(hash_rank[hash_rank==1])))
 print('Hashtags with which 0% of Tweets are disasters: ')
 print(list(hash_rank[hash_rank==0].index))
print('Total: ' + str(len(hash_rank[hash_rank==0])) )
scaled_train_X = scaled_train_X.drop(['Moving_average_700_mean'],axis=1)
 scaled_train_X = scaled_train_X.drop(['Moving_average_400_mean'],axis=1)
<load_from_csv>
vec_text = TfidfVectorizer(min_df = 10, ngram_range =(1,2), stop_words='english')
 text_vec = vec_text.fit_transform(train['text_clean'])
 text_vec_test = vec_text.transform(test['text_clean'])
 X_train_text = pd.DataFrame(text_vec.toarray() , columns=vec_text.get_feature_names())
 X_test_text = pd.DataFrame(text_vec_test.toarray() , columns=vec_text.get_feature_names())
 print(X_train_text.shape )
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 test_X = pd.DataFrame(columns=train_X.columns, dtype=np.float32, index=submission.index)
 for seg_id in tqdm_notebook(test_X.index):
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 create_features(seg_id, seg['acoustic_data'], test_X)
 scaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns )<drop_column>
train = train.join(X_train_link, rsuffix='_link')
 train = train.join(X_train_men, rsuffix='_mention')
 train = train.join(X_train_hash, rsuffix='_hashtag')
 train = train.join(X_train_text, rsuffix='_text')
 test = test.join(X_test_link, rsuffix='_link')
 test = test.join(X_test_men, rsuffix='_mention')
 test = test.join(X_test_hash, rsuffix='_hashtag')
 test = test.join(X_test_text, rsuffix='_text')
 print(train.shape, test.shape )
scaled_test_X = scaled_test_X.drop(['Moving_average_700_mean'],axis=1)
 scaled_test_X = scaled_test_X.drop(['Moving_average_400_mean'],axis=1)
<init_hyperparams>
features_to_drop = ['id', 'keyword','location','text','location_clean','text_clean', 'hashtags', 'mentions','links']
scaler = MinMaxScaler()
X_train = train.drop(columns = features_to_drop + ['target'])
 X_test = test.drop(columns = features_to_drop)
 y_train = train.target
lr = LogisticRegression(solver='liblinear', random_state=777)
 pipeline = Pipeline([('scale',scaler),('lr', lr),])
 pipeline.fit(X_train, y_train)
 y_test = pipeline.predict(X_test)
 submit = sub_sample.copy()
submit.target = y_test
submit.to_csv('submit_lr.csv',index=False )
params = {'num_leaves': 240,
'min_data_in_leaf': 180,
'objective':'regression',
'max_depth': -1,
'learning_rate': 0.001,
"boosting": "gbdt",
"feature_fraction": 0.91,
"bagging_freq": 2,
"bagging_fraction": 0.91,
"bagging_seed": 42,
"metric": 'mae',
"lambda_l1": 0.1,
"verbosity": -1,
"random_state": 42}
oof = np.zeros(len(scaled_train_X))
predictions = np.zeros(len(scaled_test_X))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(scaled_train_X,train_y.values)) :
strLog = "fold {}".format(fold_)
 print(strLog)
 X_tr, X_val = scaled_train_X.iloc[trn_idx], scaled_train_X.iloc[val_idx]
y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]
model = lgb.LGBMRegressor(**params, n_estimators = 20000, n_jobs = -1)
 model.fit(X_tr,
y_tr,
eval_set=[(X_tr, y_tr),(X_val, y_val)],
eval_metric='mae',
verbose=1000,
early_stopping_rounds=900)
 oof[val_idx] = model.predict(X_val, num_iteration=model.best_iteration_)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = train_columns
fold_importance_df["importance"] = model.feature_importances_[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += model.predict(scaled_test_X, num_iteration=model.best_iteration_)/ folds.n_splits<save_to_csv>
print('Training accuracy: %.4f' % pipeline.score(X_train, y_train))
submission.time_to_failure = predictions
submission.to_csv('submission.csv',index=True )<set_options>
print('Training f-1 score: %.4f' % f1_score(y_train, pipeline.predict(X_train)) )
warnings.filterwarnings("ignore" )<define_variables>
pd.DataFrame(confusion_matrix(y_train, pipeline.predict(X_train)) )
OUTPUT_DIR = r'd:\
DATA_DIR = r'd:\
SIG_LEN = 150000
NUM_SEG_PER_PROC = 4000
NUM_THREADS = 6
NY_FREQ_IDX = 75000
CUTOFF = 18000
MAX_FREQ_IDX = 20000
FREQ_STEP = 2500<save_to_csv>
cv = ShuffleSplit(n_splits=5, test_size=0.2, random_state=123)
 cv_score = cross_val_score(pipeline, X_train, y_train, cv=cv, scoring='f1')
 print('Cross validation F-1 score: %.3f' %np.mean(cv_score))
def split_raw_data() :
df = pd.read_csv(os.path.join(DATA_DIR, 'train.csv'))
max_start_index = len(df.index)- SIG_LEN
slice_len = int(max_start_index / 6)
 for i in range(NUM_THREADS):
print('working', i)
 df0 = df.iloc[slice_len * i:(slice_len *(i + 1)) + SIG_LEN]
df0.to_csv(os.path.join(DATA_DIR, 'raw_data_%d.csv' % i), index=False)
 del df0
del df<define_variables>
print('Optimal no.of features: %d' % np.insert(X_range, 0, 1)[np.argmax(rfecv.grid_scores_)] )
def build_rnd_idxs() :
rnd_idxs = np.zeros(shape=(NUM_THREADS, NUM_SEG_PER_PROC), dtype=np.int32)
 max_start_idx = 100000000
for i in range(NUM_THREADS):
np.random.seed(5591 + i)
 start_indices = np.random.randint(0, max_start_idx, size=NUM_SEG_PER_PROC, dtype=np.int32)
 rnd_idxs[i, :] = start_indices
for i in range(NUM_THREADS):
print(rnd_idxs[i, :8])
 print(rnd_idxs[i, -8:])
 print(min(rnd_idxs[i,:]), max(rnd_idxs[i,:]))
np.savetxt(fname=os.path.join(OUTPUT_DIR, 'start_indices_4k.csv'), X=np.transpose(rnd_idxs), fmt='%d', delimiter=',' )<normalization>
selected_features = X_train.columns[rfecv.ranking_ == 1]
X_train2 = X_train[selected_features]
X_test2 = X_test[selected_features]
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta<feature_engineering>
pipeline.fit(X_train2, y_train)
 cv2 = ShuffleSplit(n_splits=5, test_size=0.2, random_state=456)
 cv_score2 = cross_val_score(pipeline, X_train2, y_train, cv=cv2, scoring='f1')
 print('Cross validation F-1 score: %.3f' %np.mean(cv_score2))
def create_features(seg_id, seg, X, st, end):
try:
X.loc[seg_id, 'seg_id'] = np.int32(seg_id)
 X.loc[seg_id, 'seg_start'] = np.int32(st)
 X.loc[seg_id, 'seg_end'] = np.int32(end)
 except:
pass
xc = pd.Series(seg['acoustic_data'].values)
 xcdm = xc - np.mean(xc)
 b, a = des_bw_filter_lp(cutoff=18000)
 xcz = sg.lfilter(b, a, xcdm)
 zc = np.fft.fft(xcz)
 zc = zc[:MAX_FREQ_IDX]
realFFT = np.real(zc)
 imagFFT = np.imag(zc)
 freq_bands = [x for x in range(0, MAX_FREQ_IDX, FREQ_STEP)]
magFFT = np.sqrt(realFFT ** 2 + imagFFT ** 2)
 phzFFT = np.arctan(imagFFT / realFFT)
 phzFFT[phzFFT == -np.inf] = -np.pi / 2.0
phzFFT[phzFFT == np.inf] = np.pi / 2.0
phzFFT = np.nan_to_num(phzFFT)
 for freq in freq_bands:
X.loc[seg_id, 'FFT_Mag_01q%d' % freq] = np.quantile(magFFT[freq: freq + FREQ_STEP], 0.01)
 X.loc[seg_id, 'FFT_Mag_10q%d' % freq] = np.quantile(magFFT[freq: freq + FREQ_STEP], 0.1)
 X.loc[seg_id, 'FFT_Mag_90q%d' % freq] = np.quantile(magFFT[freq: freq + FREQ_STEP], 0.9)
 X.loc[seg_id, 'FFT_Mag_99q%d' % freq] = np.quantile(magFFT[freq: freq + FREQ_STEP], 0.99)
 X.loc[seg_id, 'FFT_Mag_mean%d' % freq] = np.mean(magFFT[freq: freq + FREQ_STEP])
 X.loc[seg_id, 'FFT_Mag_std%d' % freq] = np.std(magFFT[freq: freq + FREQ_STEP])
 X.loc[seg_id, 'FFT_Mag_max%d' % freq] = np.max(magFFT[freq: freq + FREQ_STEP])
 X.loc[seg_id, 'FFT_Phz_mean%d' % freq] = np.mean(phzFFT[freq: freq + FREQ_STEP])
 X.loc[seg_id, 'FFT_Phz_std%d' % freq] = np.std(phzFFT[freq: freq + FREQ_STEP])
 X.loc[seg_id, 'FFT_Rmean'] = realFFT.mean()
X.loc[seg_id, 'FFT_Rstd'] = realFFT.std()
X.loc[seg_id, 'FFT_Rmax'] = realFFT.max()
X.loc[seg_id, 'FFT_Rmin'] = realFFT.min()
X.loc[seg_id, 'FFT_Imean'] = imagFFT.mean()
X.loc[seg_id, 'FFT_Istd'] = imagFFT.std()
X.loc[seg_id, 'FFT_Imax'] = imagFFT.max()
X.loc[seg_id, 'FFT_Imin'] = imagFFT.min()
X.loc[seg_id, 'FFT_Rmean_first_6000'] = realFFT[:6000].mean()
X.loc[seg_id, 'FFT_Rstd__first_6000'] = realFFT[:6000].std()
X.loc[seg_id, 'FFT_Rmax_first_6000'] = realFFT[:6000].max()
X.loc[seg_id, 'FFT_Rmin_first_6000'] = realFFT[:6000].min()
X.loc[seg_id, 'FFT_Rmean_first_18000'] = realFFT[:18000].mean()
X.loc[seg_id, 'FFT_Rstd_first_18000'] = realFFT[:18000].std()
X.loc[seg_id, 'FFT_Rmax_first_18000'] = realFFT[:18000].max()
X.loc[seg_id, 'FFT_Rmin_first_18000'] = realFFT[:18000].min()
del xcz
del zc
b, a = des_bw_filter_lp(cutoff=2500)
 xc0 = sg.lfilter(b, a, xcdm)
 b, a = des_bw_filter_bp(low=2500, high=5000)
 xc1 = sg.lfilter(b, a, xcdm)
 b, a = des_bw_filter_bp(low=5000, high=7500)
 xc2 = sg.lfilter(b, a, xcdm)
 b, a = des_bw_filter_bp(low=7500, high=10000)
 xc3 = sg.lfilter(b, a, xcdm)
 b, a = des_bw_filter_bp(low=10000, high=12500)
 xc4 = sg.lfilter(b, a, xcdm)
 b, a = des_bw_filter_bp(low=12500, high=15000)
 xc5 = sg.lfilter(b, a, xcdm)
 b, a = des_bw_filter_bp(low=15000, high=17500)
 xc6 = sg.lfilter(b, a, xcdm)
 b, a = des_bw_filter_bp(low=17500, high=20000)
 xc7 = sg.lfilter(b, a, xcdm)
 b, a = des_bw_filter_hp(cutoff=20000)
 xc8 = sg.lfilter(b, a, xcdm)
 sigs = [xc, pd.Series(xc0), pd.Series(xc1), pd.Series(xc2), pd.Series(xc3),
pd.Series(xc4), pd.Series(xc5), pd.Series(xc6), pd.Series(xc7), pd.Series(xc8)]
for i, sig in enumerate(sigs):
X.loc[seg_id, 'mean_%d' % i] = sig.mean()
X.loc[seg_id, 'std_%d' % i] = sig.std()
X.loc[seg_id, 'max_%d' % i] = sig.max()
X.loc[seg_id, 'min_%d' % i] = sig.min()
X.loc[seg_id, 'mean_change_abs_%d' % i] = np.mean(np.diff(sig))
X.loc[seg_id, 'mean_change_rate_%d' % i] = np.mean(np.nonzero(( np.diff(sig)/ sig[:-1])) [0])
 X.loc[seg_id, 'abs_max_%d' % i] = np.abs(sig ).max()
X.loc[seg_id, 'abs_min_%d' % i] = np.abs(sig ).min()
X.loc[seg_id, 'std_first_50000_%d' % i] = sig[:50000].std()
X.loc[seg_id, 'std_last_50000_%d' % i] = sig[-50000:].std()
X.loc[seg_id, 'std_first_10000_%d' % i] = sig[:10000].std()
X.loc[seg_id, 'std_last_10000_%d' % i] = sig[-10000:].std()
X.loc[seg_id, 'avg_first_50000_%d' % i] = sig[:50000].mean()
X.loc[seg_id, 'avg_last_50000_%d' % i] = sig[-50000:].mean()
X.loc[seg_id, 'avg_first_10000_%d' % i] = sig[:10000].mean()
X.loc[seg_id, 'avg_last_10000_%d' % i] = sig[-10000:].mean()
X.loc[seg_id, 'min_first_50000_%d' % i] = sig[:50000].min()
X.loc[seg_id, 'min_last_50000_%d' % i] = sig[-50000:].min()
X.loc[seg_id, 'min_first_10000_%d' % i] = sig[:10000].min()
X.loc[seg_id, 'min_last_10000_%d' % i] = sig[-10000:].min()
X.loc[seg_id, 'max_first_50000_%d' % i] = sig[:50000].max()
X.loc[seg_id, 'max_last_50000_%d' % i] = sig[-50000:].max()
X.loc[seg_id, 'max_first_10000_%d' % i] = sig[:10000].max()
X.loc[seg_id, 'max_last_10000_%d' % i] = sig[-10000:].max()
X.loc[seg_id, 'max_to_min_%d' % i] = sig.max() / np.abs(sig.min())
 X.loc[seg_id, 'max_to_min_diff_%d' % i] = sig.max() - np.abs(sig.min())
 X.loc[seg_id, 'count_big_%d' % i] = len(sig[np.abs(sig)> 500])
 X.loc[seg_id, 'sum_%d' % i] = sig.sum()
X.loc[seg_id, 'mean_change_rate_first_50000_%d' % i] = np.mean(np.nonzero(( np.diff(sig[:50000])/ sig[:50000][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_last_50000_%d' % i] = np.mean(np.nonzero(( np.diff(sig[-50000:])/ sig[-50000:][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_first_10000_%d' % i] = np.mean(np.nonzero(( np.diff(sig[:10000])/ sig[:10000][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_last_10000_%d' % i] = np.mean(np.nonzero(( np.diff(sig[-10000:])/ sig[-10000:][:-1])) [0])
 X.loc[seg_id, 'q95_%d' % i] = np.quantile(sig, 0.95)
 X.loc[seg_id, 'q99_%d' % i] = np.quantile(sig, 0.99)
 X.loc[seg_id, 'q05_%d' % i] = np.quantile(sig, 0.05)
 X.loc[seg_id, 'q01_%d' % i] = np.quantile(sig, 0.01)
 X.loc[seg_id, 'abs_q95_%d' % i] = np.quantile(np.abs(sig), 0.95)
 X.loc[seg_id, 'abs_q99_%d' % i] = np.quantile(np.abs(sig), 0.99)
 X.loc[seg_id, 'abs_q05_%d' % i] = np.quantile(np.abs(sig), 0.05)
 X.loc[seg_id, 'abs_q01_%d' % i] = np.quantile(np.abs(sig), 0.01)
 X.loc[seg_id, 'trend_%d' % i] = add_trend_feature(sig)
 X.loc[seg_id, 'abs_trend_%d' % i] = add_trend_feature(sig, abs_values=True)
 X.loc[seg_id, 'abs_mean_%d' % i] = np.abs(sig ).mean()
X.loc[seg_id, 'abs_std_%d' % i] = np.abs(sig ).std()
X.loc[seg_id, 'mad_%d' % i] = sig.mad()
X.loc[seg_id, 'kurt_%d' % i] = sig.kurtosis()
X.loc[seg_id, 'skew_%d' % i] = sig.skew()
X.loc[seg_id, 'med_%d' % i] = sig.median()
X.loc[seg_id, 'Hilbert_mean_%d' % i] = np.abs(hilbert(sig)).mean()
X.loc[seg_id, 'Hann_window_mean'] =(convolve(xc, hann(150), mode='same')/ sum(hann(150)) ).mean()
X.loc[seg_id, 'classic_sta_lta1_mean_%d' % i] = classic_sta_lta(sig, 500, 10000 ).mean()
X.loc[seg_id, 'classic_sta_lta2_mean_%d' % i] = classic_sta_lta(sig, 5000, 100000 ).mean()
X.loc[seg_id, 'classic_sta_lta3_mean_%d' % i] = classic_sta_lta(sig, 3333, 6666 ).mean()
X.loc[seg_id, 'classic_sta_lta4_mean_%d' % i] = classic_sta_lta(sig, 10000, 25000 ).mean()
X.loc[seg_id, 'Moving_average_700_mean_%d' % i] = sig.rolling(window=700 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_1500_mean_%d' % i] = sig.rolling(window=1500 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_3000_mean_%d' % i] = sig.rolling(window=3000 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_6000_mean_%d' % i] = sig.rolling(window=6000 ).mean().mean(skipna=True)
 ewma = pd.Series.ewm
X.loc[seg_id, 'exp_Moving_average_300_mean_%d' % i] = ewma(sig, span=300 ).mean().mean(skipna=True)
 X.loc[seg_id, 'exp_Moving_average_3000_mean_%d' % i] = ewma(sig, span=3000 ).mean().mean(skipna=True)
 X.loc[seg_id, 'exp_Moving_average_30000_mean_%d' % i] = ewma(sig, span=6000 ).mean().mean(skipna=True)
 no_of_std = 2
X.loc[seg_id, 'MA_700MA_std_mean_%d' % i] = sig.rolling(window=700 ).std().mean()
X.loc[seg_id, 'MA_700MA_BB_high_mean_%d' % i] =(
 X.loc[seg_id, 'Moving_average_700_mean_%d' % i] + no_of_std * X.loc[seg_id, 'MA_700MA_std_mean_%d' % i] ).mean()
X.loc[seg_id, 'MA_700MA_BB_low_mean_%d' % i] =(
 X.loc[seg_id, 'Moving_average_700_mean_%d' % i] - no_of_std * X.loc[seg_id, 'MA_700MA_std_mean_%d' % i] ).mean()
X.loc[seg_id, 'MA_400MA_std_mean_%d' % i] = sig.rolling(window=400 ).std().mean()
X.loc[seg_id, 'MA_400MA_BB_high_mean_%d' % i] =(
 X.loc[seg_id, 'Moving_average_700_mean_%d' % i] + no_of_std * X.loc[seg_id, 'MA_400MA_std_mean_%d' % i] ).mean()
X.loc[seg_id, 'MA_400MA_BB_low_mean_%d' % i] =(
 X.loc[seg_id, 'Moving_average_700_mean_%d' % i] - no_of_std * X.loc[seg_id, 'MA_400MA_std_mean_%d' % i] ).mean()
X.loc[seg_id, 'MA_1000MA_std_mean_%d' % i] = sig.rolling(window=1000 ).std().mean()
X.loc[seg_id, 'iqr_%d' % i] = np.subtract(*np.percentile(sig, [75, 25]))
X.loc[seg_id, 'q999_%d' % i] = np.quantile(sig, 0.999)
 X.loc[seg_id, 'q001_%d' % i] = np.quantile(sig, 0.001)
 X.loc[seg_id, 'ave10_%d' % i] = stats.trim_mean(sig, 0.1)
 for windows in [10, 100, 1000]:
x_roll_std = xc.rolling(windows ).std().dropna().values
x_roll_mean = xc.rolling(windows ).mean().dropna().values
X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(
 np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(
 np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()
return X<load_from_csv>
grid={"C":np.logspace(-2,2,5), "penalty":["l1","l2"]}
lr_cv = GridSearchCV(LogisticRegression(solver='liblinear', random_state=20), grid, cv=cv2, scoring = 'f1')
 pipeline_grid = Pipeline([('scale',scaler),('gridsearch', lr_cv),])
 pipeline_grid.fit(X_train2, y_train)
 print("Best parameter: ", lr_cv.best_params_)
 print("F-1 score: %.3f" %lr_cv.best_score_ )
def build_fields(proc_id):
success = 1
count = 0
try:
seg_st = int(NUM_SEG_PER_PROC * proc_id)
 train_df = pd.read_csv(os.path.join(DATA_DIR, 'raw_data_%d.csv' % proc_id), dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})
 len_df = len(train_df.index)
 start_indices =(np.loadtxt(fname=os.path.join(OUTPUT_DIR, 'start_indices_4k.csv'), dtype=np.int32, delimiter=',')) [:, proc_id]
train_X = pd.DataFrame(dtype=np.float64)
 train_y = pd.DataFrame(dtype=np.float64, columns=['time_to_failure'])
 t0 = time.time()
for seg_id, start_idx in zip(range(seg_st, seg_st + NUM_SEG_PER_PROC), start_indices):
end_idx = np.int32(start_idx + 150000)
 print('working: %d, %d, %d to %d of %d' %(proc_id, seg_id, start_idx, end_idx, len_df))
seg = train_df.iloc[start_idx: end_idx]
train_X = create_features(seg_id, seg, train_X, start_idx, end_idx)
 train_y.loc[seg_id, 'time_to_failure'] = seg['time_to_failure'].values[-1]
if count == 10:
print('saving: %d, %d to %d' %(seg_id, start_idx, end_idx))
train_X.to_csv('train_x_%d.csv' % proc_id, index=False)
 train_y.to_csv('train_y_%d.csv' % proc_id, index=False)
 count += 1
print('final_save, process id: %d, loop time: %.2f for %d iterations' %(proc_id, time.time() - t0, count))
train_X.to_csv(os.path.join(OUTPUT_DIR, 'train_x_%d.csv' % proc_id), index=False)
 train_y.to_csv(os.path.join(OUTPUT_DIR, 'train_y_%d.csv' % proc_id), index=False)
 except:
print(traceback.format_exc())
 success = 0
return success<categorify>
y_test2 = pipeline_grid.predict(X_test2)
 submit2 = sub_sample.copy()
submit2.target = y_test2
submit2.to_csv('submit_lr2.csv',index=False )
def run_mp_build() :
t0 = time.time()
num_proc = NUM_THREADS
pool = mp.Pool(processes=num_proc)
 results = [pool.apply_async(build_fields, args=(pid,)) for pid in range(NUM_THREADS)]
output = [p.get() for p in results]
num_built = sum(output)
 pool.close()
pool.join()
print(num_built)
 print('Run time: %.2f' %(time.time() - t0))<load_from_csv>
y_hat = pipeline_grid.predict_proba(X_train2)[:,1]
checker = train.loc[:,['text','keyword','location','target']]
checker['pred_prob'] = y_hat
checker['error'] = np.abs(checker['target'] - checker['pred_prob'])
 error50 = checker.sort_values('error', ascending=False ).head(50)
 error50 = error50.rename_axis('id' ).reset_index()
error50.target.value_counts()
<load_from_csv><EOS> 
pd.options.display.max_colwidth = 200
error50.loc[0:10,['text','target','pred_prob']]