Kaggle competition: Quora Question Pairs
pred = np.zeros(len(X_train))
pred[:len(pred2)] += pred2
pred[len(pred2):] += pred1
print(f"Half/Half RMSE: {rmse(pred, y_train):.4f}" )<load_pretrained>
df_train = pd.read_csv('.. /input/train.csv',encoding='utf8')[:10000]
df_train = df_train.fillna('leeg')
 df_test = pd.read_csv('.. /input/test.csv',encoding='utf8')[:50000]
df_test = df_test.fillna('leeg')
 df_train.head(2)
 df_test.head(2)
 end = time.clock()
print('open:',end-start )
pickle.dump(model_half_1, open("model_half_1.pkl", "wb"))
pickle.dump(model_half_2, open("model_half_2.pkl", "wb"))
pickle.dump(pred, open("pred_L1.pkl", "wb"))<drop_column>
stop = set(stopwords.words('english'))
rake_object = rake.Rake(stop, 3, 3, 1)
 text = "Natural language processing(NLP)deals with the application of computational models to text or speech data.Application areas within NLP include automatic(machine)translation between languages; dialogue systems, which allow a human to interact with a machine using natural language; and information extraction, where the goal is to transform unstructured text into structured(database)representations that can be searched and browsed in flexible ways.NLP technologies are having a dramatic impact on the way people interact with computers, on the way people interact with each other through the use of language, and on the way people access the vast amount of linguistic data now in electronic form.From a scientific viewpoint, NLP involves fundamental questions of how to structure formal models(for example statistical models)of natural language phenomena, and of how to design algorithms that implement these models.In this course you will study mathematical and computational models of language, and the application of these models to key problems in natural language processing.The course has a focus on machine learning methods, which are widely used in modern NLP systems: we will cover formalisms such as hidden Markov models, probabilistic context-free grammars, log-linear models, and statistical models for machine translation.The curriculum closely follows a course currently taught by Professor Collins at Columbia University, and previously taught at MIT."
keywords = rake_object.run(text)
 print("keywords: ", keywords)
del X_train, X_half_1, X_half_2, y_half_1, y_half_2, d_half_1, d_half_2,
del watchlist_1, watchlist_2, pred, pred1, pred2
gc.collect()<load_from_csv>
def cleantxt(x):
x = str(x)
 for char in ['.', '"', ',', '(', ')', '!', '?', ';', ':']:
x = x.replace(char, ' ')
 return x
def cleantxtsplit(x):
x=cleantxt(x)
 return x.split()
def word_match_share(row):
q1words = {}
q2words = {}
for word in cleantxtsplit(row['question1']):
q1words[word] = 1
for word in cleantxtsplit(row['question2']):
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
nonshared_words_in_q1 = [w for w in q1words.keys() if w not in q2words]
nonshared_words_in_q2 = [w for w in q2words.keys() if w not in q1words]
X1 =(len(shared_words_in_q1)) /(len(q1words))
Y1 =(len(nonshared_words_in_q1)) /(len(q1words))
X2 =(len(shared_words_in_q2)) /(len(q2words))
Y2 =(len(nonshared_words_in_q2)) /(len(q2words))
R3 =(Y1+Y2)/2
return R3
train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
 train_qs = pd.Series(df_train['question1'].map(cleantxt ).tolist() + df_train['question2'].map(cleantxt ).tolist() ).astype(str)
 test_qs = pd.Series(df_test['question1'].map(cleantxt ).tolist() + df_test['question2'].map(cleantxt ).tolist() ).astype(str)
 end = time.clock()
print('clean:',end-start )
weather_test = weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv' )<feature_engineering>
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }
end = time.clock()
print('def:',end-start )
%%time
df_test = pd.read_feather('.. /input/ashrae-feather/test.ft')
 weather_test = fill_weather_dataset(weather_test)
 df_test['hour'] = df_test['timestamp'].dt.hour
df_test['weekday'] = df_test['timestamp'].dt.weekday
df_test['bm'] = df_test['building_id'].apply(lambda x: str(x)) + '_' + df_test['meter'].apply(lambda x: str(x))
df_test = df_test.merge(bm, right_index=True, left_on=bm_cols, how='left')
 df_test.drop('bm', axis=1, inplace=True)
 df_test = reduce_mem_usage(df_test, use_float16=True)
 weather_test = reduce_mem_usage(weather_test, use_float16=True)
 X_test, row_ids = prepare_data(df_test, building, weather_test, test=True )<set_options>
format = lambda x: x.split(' ', 1)[0]
merknamen=df_train['question1'].map(format)
 merknamen=list(set(merknamen))
txt1=df_train['question1']
txt2=df_train['question2']
txt=txt1.append(pd.DataFrame(list(txt2)))
 txt.columns=['q']
print(txt.head())
 tfidf = TfidfVectorizer(ngram_range=(4,4),min_df=1,stop_words=merknamen)
 tfidf.fit(txt.q)
 tfidf_m = tfidf.transform(txt.q)
 words = pd.DataFrame(pd.Series(list(tfidf.vocabulary_.keys()), index=tfidf.vocabulary_.values()),columns=['woord'])
del df_test, building, weather_test
gc.collect()<predict_on_test>
def sort_coo(m):
tuples = izip(m.row, m.col, m.data)
 return sorted(tuples, key=lambda x:(x[2]))
txt['core']=''
for xi in range(0,len(txt)) :
rij1=tfidf_m[xi][0:]
print(sort_coo(coo_matrix(rij1)))
 rij1['woord']=words['woord']
toprij=rij1[rij1['plaats']>0]
formathw = lambda x: len(x)
 toprij['lengte']=toprij['woord'].map(formathw)
 toprij=toprij[toprij['lengte']==toprij['lengte'].max() ]['woord']
if len(toprij)>0:
print(list(toprij)[0])
 txt.set_value(xi, 'klas', list(toprij)[0])
 print(txt)
 end = time.clock()
print('tf:',end-start )
%%time
pred = np.expm1(model_half_1.predict(X_test, num_iteration=model_half_1.best_iteration)) / 2
del model_half_1
gc.collect()
pred += np.expm1(model_half_2.predict(X_test, num_iteration=model_half_2.best_iteration)) / 2
del model_half_2
gc.collect()<save_to_csv>
print('Most common words and weights:
')
 print(sorted(weights.items() , key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
 print('
Least common words and weights: ')
(sorted(weights.items() , key=lambda x: x[1], reverse=True)[:10] )
submission = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(pred, 0, a_max=None)})
 submission['meter_reading'] = submission['meter_reading'].astype('float32')
 submission['row_id'] = submission['row_id'].astype('int32')
 submission.to_csv("submission.csv", index=False, chunksize=25000 )<filter>
def tf(word,blob):
return blob.words.count(word)/len(blob.words)
 def n_containing(word,bloblist):
return blob.words.count(word)/len(blob.words)
 def idf(word,bloblist):
return sum(1 for blob in bloblist if word in blob)
 def tfidf(word,blob,bloblist):
return tf(word,blob)*idf(word,bloblist)
 def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).split() :
q1words[word] = 1
for word in str(row['question2'] ).split() :
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights_q1=[weights.get(w, 0)for w in q1words.keys() if w in q2words]
shared_weights_q2=[weights.get(w, 0)for w in q2words.keys() if w in q1words]
nonshared_weights_q1 = [weights.get(w, 0)for w in q1words.keys() if w not in q2words]
nonshared_weights_q2 = [weights.get(w, 0)for w in q2words.keys() if w not in q1words]
total_weights_q1 = [weights.get(w, 0)for w in q1words]
total_weights_q2 =[weights.get(w, 0)for w in q2words]
X1 = np.sum(shared_weights_q1)/ np.sum(total_weights_q1)
 Y1 = np.sum(nonshared_weights_q1)/ np.sum(total_weights_q1)
 X2 = np.sum(shared_weights_q2)/ np.sum(total_weights_q2)
 Y2 = np.sum(nonshared_weights_q2)/ np.sum(total_weights_q2)
 R3 =(Y1+Y2)/2
return R3
end = time.clock()
print('wordmatch:',end-start )
submission[X_test['building_id'] > 105]<load_from_disk>
tfidf_train_word_match = df_train.apply(tfidf_word_match_share, axis=1, raw=True)
 end = time.clock()
print('tfidf:',end-start )
site0 = pd.read_feather('.. /input/ucf-building-meter-reading/site0.ft')
 df_test = pd.read_feather('.. /input/ashrae-feather/test.ft' )<merge>
print('Original AUC:', roc_auc_score(df_train['is_duplicate'], train_word_match))
print(' TFIDF AUC:', roc_auc_score(df_train['is_duplicate'], tfidf_train_word_match.fillna(0)) )
merged = df_test.merge(site0, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )<prepare_x_and_y>
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)
 y_train = df_train['is_duplicate'].values
end = time.clock()
print('createtestdata:',end-start )
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']<compute_test_metric>
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
p = 0.165
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -=1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print(len(pos_train)/(len(pos_train)+ len(neg_train)))
 x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train
print(f'RMSLE of buildings 0-104: {rmsle(ytrue, pred):.4f}' )<load_from_disk>
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242 )
site1 = pd.read_feather('.. /input/ucl-data-leakage-episode-2/site1.ft')
 site1 = site1.query('timestamp >= 2017' )<merge>
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
bst = xgb.train(params, d_train, 400, watchlist, early_stopping_rounds=50, verbose_eval=10)
merged = df_test.merge(site1, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )<prepare_x_and_y>
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False )