Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<compute_test_metric>
tqdm.pandas()
gc.collect()
def confidence_out(y_pred):
confidence = np.zeros(y_pred.shape[0])
 for i in range(len(confidence)) :
if y_pred[i,:].argmax() != 1:
confidence[i] = y_pred[i,2]-y_pred[i,0]
return confidence<predict_on_test>
max_features= 200000
max_senten_len = 40
max_senten_num = 3
embed_size = 300
VALIDATION_SPLIT = 0
y_pred_proba = lgb_clf.predict_proba(X_val)
 predicted_return = y_pred_proba[:,1] - y_pred_proba[:,0]
predicted_return = rescale(predicted_return, r_train )<compute_test_metric>
from sklearn.utils import shuffle
r_val = r_val.clip(-1,1)
 x_t_i = predicted_return * r_val * u_val
data = {'day' : d_val, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print('Validation score', score_valid )<split>
df = pd.read_csv('.. /input/train.csv' )
clfs = []
for i in range(20):
clf = lgb.LGBMClassifier(learning_rate=0.1, random_state=1200+i, silent=True,
n_jobs=4, n_estimators=2500)
 clf.set_params(**opt_params)
 clfs.append(( 'lgbm%i'%i, clf))
def split_data(X, y, test_percentage=0.2, seed=None):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_percentage)
 return X_train, y_train, X_test, y_test
def _parallel_fit_estimator(estimator, X, y, sample_weight=None, **fit_params):
X_train, y_train, X_test, y_test = split_data(X, y, seed=1992)
 fit_params["eval_set"] = [(X_train,y_train),(X_test,y_test)]
if sample_weight is not None:
estimator.fit(X_train, y_train, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X_train, y_train, **fit_params)
 return estimator<train_model>
test_df = pd.read_csv(".. /input/test.csv" )
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, **fit_params)
 for clf in clfs if clf is not None)
 return self<train_model>
len(df.target.unique() )
vc = VotingClassifierLGBM(clfs, voting='soft')
 vc.fit(X_train, y_train, **fit_params)
 filename = 'VotingClassifierLGBM.sav'
pickle.dump(vc, open(filename, 'wb'))<predict_on_test>
df.columns = ['qid', 'text', 'category']
test_df.columns = ['qid', 'text']
vc = pickle.load(open(filename, 'rb'))
vc.voting = 'soft'
predicted_class = vc.predict(X_val)
 predicted_return = vc.predict_proba(X_val)
 predicted_return = vc.predict_proba(X_val)[:,1]*2-1
predicted_return = rescale(predicted_return, r_train )<compute_test_metric>
df = df[['text', 'category']]
vc.voting = 'soft'
global_accuracy_soft = accuracy_score(y_val, predicted_class)
 global_f1_soft = f1_score(y_val, predicted_class)
 print('Accuracy score clfs: %f' % global_accuracy_soft)
 print('F1 score clfs: %f' % global_f1_soft )<compute_test_metric>
df['text'] = df['text'].str.lower()
test_df['text'] = test_df['text'].str.lower()
r_val = r_val.clip(-1,1)
 x_t_i = predicted_return * r_val * u_val
data = {'day' : d_val, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print('Validation score', score_valid )<feature_engineering>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", 'u.s':'america', 'e.g':'for example'}
days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if n_days % 50 == 0:
print(n_days,end=' ')
 t = time.time()
column_market = ['returnsClosePrevMktres1','returnsOpenPrevMktres1','returnsClosePrevMktres10', 'returnsOpenPrevMktres10']
column_raw = ['returnsClosePrevRaw1', 'returnsOpenPrevRaw1','returnsClosePrevRaw10', 'returnsOpenPrevRaw10']
market_obs_df['close_open_ratio'] = np.abs(market_obs_df['close']/market_obs_df['open'])
 for i in range(len(column_raw)) :
market_obs_df[column_market[i]] = market_obs_df[column_market[i]].fillna(market_obs_df[column_raw[i]])
 market_obs_df = market_obs_df[market_obs_df.assetCode.isin(predictions_template_df.assetCode)]
market_obs_df = market_obs_df[market_obs_df.assetCode.isin(asset_code_dict.keys())]
market_obs = data_prep(market_obs_df, news_obs_df)
 market_obs[num_columns] = data_scaler.transform(market_obs[num_columns])
 X_live = market_obs[feature_columns].values
prep_time += time.time() - t
t = time.time()
lp = vc.predict_proba(X_live)
 prediction_time += time.time() -t
t = time.time()
confidence = lp[:,1] - lp[:,0]
confidence = rescale(confidence, r_train)
 preds = pd.DataFrame({'assetCode':market_obs['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()<train_model>
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data()<merge>
df['text'] = df['text'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))
test_df['text'] = test_df['text'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))
def data_prep(market_train,news_train):
market_train.time = market_train.time.dt.date
news_train.time = news_train.time.dt.hour
news_train.sourceTimestamp= news_train.sourceTimestamp.dt.hour
news_train.firstCreated = news_train.firstCreated.dt.date
news_train['assetCodesLen'] = news_train['assetCodes'].map(lambda x: len(eval(x)))
 news_train['assetCodes'] = news_train['assetCodes'].map(lambda x: list(eval(x)) [0])
 kcol = ['firstCreated', 'assetCodes']
news_train = news_train.groupby(kcol, as_index=False ).mean()
market_train = pd.merge(market_train, news_train, how='left', left_on=['time', 'assetCode'],
right_on=['firstCreated', 'assetCodes'])
 lbl = {k: v for v, k in enumerate(market_train['assetCode'].unique())}
market_train['assetCodeT'] = market_train['assetCode'].map(lbl)
 market_train = market_train.dropna(axis=0)
 return market_train<categorify>
punct = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
market_train = data_prep(market_train,news_train )<filter>
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', '!':' '}
up = market_train.returnsOpenNextMktres10 >= 0<define_variables>
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
fcol = [c for c in market_train if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp']]<prepare_x_and_y>
df['text'] = df['text'].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))
test_df['text'] = test_df['text'].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))
X = market_train[fcol].values
up = up.values
r = market_train.returnsOpenNextMktres10.values<normalization>
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng )<split>
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
X_train, X_test, up_train, up_test, r_train, r_test\
= model_selection.train_test_split(X, up, r, test_size=0.25, random_state=99 )<import_modules>
df['text'] = df['text'].progress_apply(lambda x: correct_spelling(x, mispell_dict))
test_df['text'] = test_df['text'].progress_apply(lambda x: correct_spelling(x, mispell_dict))
from xgboost import XGBClassifier
import time<choose_model_class>
labels = df['category']
text = df['text']
xgb_up = XGBClassifier(n_jobs=4,n_estimators=250,max_depth=8,eta=0.1 )<train_model>

 train_text = text.reset_index().drop('index', axis=1)
 y_train = labels.reset_index().drop('index', axis=1)
 val_text = None
y_val = None
t = time.time()
print('Fitting Up')
 xgb_up.fit(X_train,up_train)
 print(f'Done, time = {time.time() - t}' )<compute_test_metric>
test = test_df['text']
accuracy_score(xgb_up.predict(X_test),up_test )<define_variables>
cates = df.groupby('category')
 print("total categories:", cates.ngroups)
 print(cates.size() )
days = env.get_prediction_days()<predict_on_test>
paras = []
labels = []
texts = []
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
print(n_days,end=' ')
 t = time.time()
market_obs_df = data_prep(market_obs_df, news_obs_df)
 market_obs_df = market_obs_df[market_obs_df.assetCode.isin(predictions_template_df.assetCode)]
X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp = xgb_up.predict_proba(X_live)
 prediction_time += time.time() -t
t = time.time()
confidence = 2* lp[:,1] -1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t<save_to_csv>
sent_lens = []
sent_nums = []
for idx in tqdm(range(train_text.shape[0])) :
text = train_text.text[idx]
texts.append(text)
 sentences = tokenize.sent_tokenize(text)
 sent_nums.append(len(sentences))
for sent in sentences:
sent_lens.append(len(text_to_word_sequence(sent)))
 paras.append(sentences )
env.write_submission_file()<train_model>
val_paras = []
val_labels = []
total = prep_time + prediction_time + packaging_time
print(f'Preparing Data: {prep_time:.2f}s')
 print(f'Making Predictions: {prediction_time:.2f}s')
 print(f'Packing: {packaging_time:.2f}s')
 print(f'Total: {total:.2f}s' )<set_options>
test_paras = []
test_labels = []
%matplotlib inline 
<import_modules>
for idx in range(test.shape[0]):
text = test[idx]
sentences = tokenize.sent_tokenize(text)
 test_paras.append(sentences )
env = twosigmanews.make_env()<define_variables>
tokenizer = Tokenizer(num_words=max_features, oov_token=True)
 tokenizer.fit_on_texts(texts )
sampling = True<split>
x_train = np.zeros(( len(texts), max_senten_num, max_senten_len), dtype='int32')
 for i, sentences in tqdm(enumerate(paras)) :
tokenized_sent = tokenizer.texts_to_sequences(sentences)
 padded_seq = pad_sequences(tokenized_sent, maxlen=max_senten_len, padding='post', truncating='post')
 for j, seq in enumerate(padded_seq):
if(j < max_senten_num):
x_train[i,j,:] = seq
else:
break
( market_train_df, news_train_df)= env.get_training_data()
if sampling:
market_train_df = market_train_df.tail(40_000)
 news_train_df = news_train_df.tail(100_000)
 else:
market_train_df = market_train_df.tail(1_500_000)
 news_train_df = news_train_df.tail(4_000_000 )<define_variables>
news_agg_cols = [f for f in news_train_df.columns if 'novelty' in f or
'volume' in f or
'sentiment' in f or
'bodySize' in f or
'Count' in f or
'marketCommentary' in f or
'relevance' in f]
news_agg_dict = {}
for col in news_agg_cols:
news_agg_dict[col] = ['mean', 'sum', 'max', 'min']
news_agg_dict['urgency'] = ['min', 'count']
news_agg_dict['takeSequence'] = ['max']<data_type_conversions>
test_data = np.zeros(( test.shape[0], max_senten_num, max_senten_len), dtype='int32')
 for i, sentences in enumerate(test_paras):
tokenized_sent = tokenizer.texts_to_sequences(sentences)
 padded_seq = pad_sequences(tokenized_sent, maxlen=max_senten_len, padding='post', truncating='post')
 for j, seq in enumerate(padded_seq):
if(j < max_senten_num):
test_data[i,j,:] = seq
else:
break
def check_index(index, indecies):
if index in indecies:
return True
else:
return False
def fillnulls(X):
X['headline'] = X['headline'].fillna('null')
 def generalize_time(X):
X['time'] = X['time'].dt.strftime('%Y-%m-%d %H:%M:%S' ).str.slice(0,16)
 def partial_groupby(market_df, news_df, df_assetCodes):
temp_news_df_expanded = pd.merge(df_assetCodes, news_df[['time', 'assetCodes']], left_on='level_0', right_index=True, suffixes=(['','_old']))
temp_news_df = temp_news_df_expanded.copy() [['time', 'assetCode']]
temp_market_df = market_df.copy() [['time', 'assetCode']]
temp_news_df['news_index'] = temp_news_df.index.values
temp_market_df['market_index'] = temp_market_df.index.values
temp_news_df.set_index(['time', 'assetCode'], inplace=True)
 temp_market_df_2 = temp_market_df.join(temp_news_df, on=['time', 'assetCode'])
 del temp_market_df, temp_news_df
temp_market_df_2 = temp_market_df_2.dropna()
market_valid_indecies = temp_market_df_2['market_index'].tolist()
news_valid_indecies = temp_market_df_2['news_index'].tolist()
del temp_market_df_2
market_df = market_df.loc[market_valid_indecies]
return news_valid_indecies
def join_market_news(market_df, news_df, nulls=False):
generalize_time(market_df)
 generalize_time(news_df)
 news_df['assetCodes'] = news_df['assetCodes'].str.findall(f"'([\w\./]+)'")
 assetCodes_expanded = list(chain(*news_df['assetCodes']))
assetCodes_index = news_df.index.repeat(news_df['assetCodes'].apply(len))
assert len(assetCodes_index)== len(assetCodes_expanded)
 df_assetCodes = pd.DataFrame({'level_0': assetCodes_index, 'assetCode': assetCodes_expanded})
 if not nulls:
news_valid_indecies = partial_groupby(market_df, news_df, df_assetCodes)
 news_col = ['time', 'assetCodes', 'headline'] + sorted(list(news_agg_dict.keys()))
news_df_expanded = pd.merge(df_assetCodes, news_df[news_col], left_on='level_0', right_index=True, suffixes=(['','_old']))
if not nulls:
news_df_expanded = news_df_expanded.loc[news_valid_indecies]
def news_df_feats(x):
if x.name == 'headline':
return list(x)
 news_df_expanded = news_df_expanded.reset_index()
news_groupby = news_df_expanded.groupby(['time', 'assetCode'])
 news_df_aggregated = news_groupby.agg(news_agg_dict ).apply(np.float32 ).reset_index()
news_df_aggregated.columns = ['_'.join(col ).strip() for col in news_df_aggregated.columns.values]
news_df_cat = news_groupby.transform(lambda x: news_df_feats(x)) ['headline'].to_frame()
new_news_df = pd.concat([news_df_aggregated, news_df_cat], axis=1)
 del news_df_aggregated
del news_df_cat
del news_df
new_news_df.rename(columns={'time_': 'time', 'assetCode_': 'assetCode'}, inplace=True)
 new_news_df.set_index(['time', 'assetCode'], inplace=True)
 market_df = market_df.join(new_news_df, on=['time', 'assetCode'])
 fillnulls(market_df)
 return market_df 
<merge>
word_index = tokenizer.word_index
print('Total %s unique tokens.' % len(word_index))
%%time
X_train = join_market_news(market_train_df, news_train_df, nulls=False )<define_variables>
import os
def round_scores(x):
if x >= 0:
return 1
else:
return 0
def get_headline_df(X_train):
headlines_lst = []
target_lst = []
for row in range(0,len(X_train.index)) :
for sentence in X_train['headline'].iloc[row]:
headlines_lst.append(sentence)
 target_lst.append(round_scores(X_train['returnsOpenNextMktres10'].iloc[row]))
return pd.DataFrame({'headline':pd.Series(headlines_lst), 'returnsOpenNextMktres10':pd.Series(target_lst)})
 def get_headline(headlines_df):
headlines_lst = []
for row in range(0,len(headlines_df.index)) :
headlines_lst.append(headlines_df.iloc[row])
 basicvectorizer = CountVectorizer()
headlines_vectorized = basicvectorizer.fit_transform(headlines_lst)
 print(headlines_vectorized.shape)
 return headlines_vectorized, basicvectorizer
def headline_mapping(target, headlines_vectored, headline_vectorizer):
print(np.asarray(target ).shape)
 headline_model = LogisticRegression()
headline_model = headline_model.fit(headlines_vectored, target)
 basicwords = headline_vectorizer.get_feature_names()
basiccoeffs = headline_model.coef_.tolist() [0]
coeff_df = pd.DataFrame({'Word' : basicwords,
'Coefficient' : basiccoeffs})
 coefficient_dict = dict(zip(coeff_df.Word, coeff_df.Coefficient))
return coefficient_dict, coeff_df['Coefficient'].mean()
def get_coeff_col(X, coeff_dict, coeff_default):
def get_coeff(word_lst):
coeff_sum = 0
for word in word_lst:
if word in coeff_dict:
coeff_sum += coeff_dict[word]
else:
coeff_sum += coeff_default
coeff_score = coeff_sum / len(word_lst)
 return coeff_score
basicvectorizer = CountVectorizer()
headlines_coeff_lst = []
for row in range(0,len(X['headline'].index)) :
coeff_score = 0
for i in range(0,len(X['headline'].iloc[row])) :
coeff_score += get_coeff(str(X['headline'].iloc[row][i] ).split(' '))
headlines_coeff_lst.append(coeff_score / len(X['headline'].iloc[row]))
coeff_mean_df = pd.DataFrame({'headline_coeff_mean': pd.Series(headlines_coeff_lst)})
 X = pd.concat([X.reset_index() , coeff_mean_df], axis=1)
 return X<categorify>
gc.collect()
word_index = tokenizer.word_index
max_features = len(word_index)+1
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word.lower() , np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word.lower() , np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
headline_df = get_headline_df(X_train)
 coefficient_dict, coefficient_default = headline_mapping(headline_df['returnsOpenNextMktres10'],
*get_headline(headline_df['headline']))<compute_train_metric>
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix = np.mean(( embedding_matrix_1, embedding_matrix_3), axis=0)
 del embedding_matrix_1, embedding_matrix_3
gc.collect()
np.shape(embedding_matrix )
X_train = get_coeff_col(X_train, coefficient_dict, coefficient_default )<feature_engineering>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
def extra_features(X):
new_col = X["close"] - X["open"]
X.insert(loc=6, column="daily_diff", value=new_col)
 X['close_to_open'] = np.abs(X['close'] / X['open'] )<feature_engineering>
def han_model(embedding_matrix):
nb_words = embedding_matrix.shape[0]
embedding_layer = Embedding(nb_words, embed_size, weights=[embedding_matrix])
 word_input = Input(shape=(max_senten_len,), dtype='float32')
 word_sequences = embedding_layer(word_input)
 word_lstm = Bidirectional(CuDNNLSTM(64, return_sequences=True))(word_sequences)
 word_att = AttentionWithContext()(word_lstm)
 word_att = ReLU()(word_att)
 wordEncoder = Model(word_input, word_att)
 sent_input = Input(shape=(max_senten_num, max_senten_len), dtype='float32')
 sent_encoder = TimeDistributed(wordEncoder )(sent_input)
 sent_enc_avg = Lambda(lambda x: K.sum(x, axis=1))(sent_encoder)
 sent_enc_avg = ReLU()(sent_enc_avg)
 pred1 = Dense(1, activation='sigmoid' )(sent_enc_avg)
 sent_lstm = Bidirectional(CuDNNLSTM(128, return_sequences=True))(sent_encoder)
 sent_att = AttentionWithContext()(sent_lstm)
 sent_att = ReLU()(sent_att)
 pred2 = Dense(1, activation='sigmoid' )(sent_att)
 preds = Average()([pred1, pred2])
 model = Model(sent_input, preds)
 model.compile(loss='binary_crossentropy',optimizer=Adam() ,metrics=[f1])
 return model
extra_features(X_train )<feature_engineering>
def train_pred(model, train_X, train_y, val_X, val_y, epochs=2, callback=None, batch_size=512):
print(train_X.dtype, train_y.dtype)
 h = model.fit(train_X, train_y, batch_size=batch_size, epochs=epochs, validation_data=(val_X, val_y), callbacks = callback, verbose=1)
 model.load_weights(filepath)
 pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 pred_test_y = model.predict([test_data], batch_size=1024, verbose=0)
 print('=' * 60)
 return pred_val_y, pred_test_y
def split_time(df):
df['time_day'] = df['time'].str.slice(8,10)
 df['time_month'] = df['time'].str.slice(5,7)
 df['time_year'] = df['time'].str.slice(0,4)
 df['time_hour'] = df['time'].str.slice(11,13)
 df['time_minute'] = df['time'].str.slice(14,16)
 df['temp_time'] = df['time'].str.replace(" UTC", "")
 df['temp_time'] = pd.to_datetime(df['temp_time'], format='%Y-%m-%d %H')
 df['time_day_of_year'] = df.temp_time.dt.dayofyear
df['time_week_of_year'] = df.temp_time.dt.weekofyear
df["time_weekday"] = df.temp_time.dt.weekday
df["time_quarter"] = df.temp_time.dt.quarter
del df['temp_time']
gc.collect()
time_feats = ['time_day', 'time_month', 'time_year']
df[time_feats] = df[time_feats].apply(pd.to_numeric)
 cal = USFederalHolidayCalendar()
holidays = cal.holidays(start='2007-01-01', end='2018-09-27' ).to_pydatetime()
df['on_holiday'] = df['time'].str.slice(0,10 ).apply(lambda x: 1 if x in holidays else 0 )<split>
from sklearn.model_selection import GridSearchCV, StratifiedKFold
split_time(X_train )<drop_column>
search_result = threshold_search(y_train, train_meta )
def remove_cols(X):
del_cols = [f for f in X.columns if X[f].dtype == 'object'] + ['assetName', 'index']
for f in del_cols:
del X[f]<drop_column>
pred_test_y =(test_meta>search_result['threshold'] ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )
<split><EOS> 
gc.collect()