Kaggle competition: Quora Question Pairs
<SOS> metric: LogLoss Kaggle data source: quora-question-pairs<categorify>
eng_stopwords = set(stopwords.words('english'))
color = sns.color_palette()
%matplotlib inline
pd.options.mode.chained_assignment = None
df_train = reduce_mem_usage(df_train, use_float16=True)
 building = reduce_mem_usage(building, use_float16=True)
 weather_train = reduce_mem_usage(weather_train, use_float16=True )<compute_test_metric>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print(train_df.shape)
 print(test_df.shape )
def rmse(ytrue, ypred):
return np.sqrt(np.mean(np.square(ypred - ytrue), axis=0))
def rmsle(ytrue, ypred):
return np.sqrt(np.mean(np.square(np.log1p(ypred)- np.log1p(ytrue)) , axis=0))<categorify>
all_ques_df = pd.DataFrame(pd.concat([train_df['question1'], train_df['question2']]))
all_ques_df.columns = ["questions"]
all_ques_df["num_of_words"] = all_ques_df["questions"].apply(lambda x : len(str(x ).split()))
le = LabelEncoder()
def prepare_data(X, building_data, weather_data, test=False):

 weather_data = timestamp_align(weather_data)
 X = X.merge(building_data, on="building_id", how="left")
 X = X.merge(weather_data, on=["site_id", "timestamp"], how="left")
 X.sort_values("timestamp")
 X.reset_index(drop=True)
 gc.collect()
X.timestamp = pd.to_datetime(X.timestamp, format="%Y-%m-%d %H:%M:%S")
 X.square_feet = np.log1p(X.square_feet)
 X["hour"] = X.timestamp.dt.hour
X["weekday"] = X.timestamp.dt.weekday
X['hour'] = X['hour'].astype('int8')
 X['weekday'] = X['weekday'].astype('int8')
 drop_features = ["timestamp", "site_id","sea_level_pressure", "precip_depth_1_hr",]
X.drop(drop_features, axis=1, inplace=True)
 X = reduce_mem_usage(X, use_float16=True)
 if test:
row_ids = X.row_id
X.drop("row_id", axis=1, inplace=True)
 return X, row_ids
else:
y = np.log1p(X.meter_reading)
 X.drop("meter_reading", axis=1, inplace=True)
 return X, y<prepare_x_and_y>
def get_unigrams(que):
return [word for word in word_tokenize(que.lower())if word not in eng_stopwords]
def get_common_unigrams(row):
return len(set(row["unigrams_ques1"] ).intersection(set(row["unigrams_ques2"])))
 def get_common_unigram_ratio(row):
return float(row["unigrams_common_count"])/ max(len(set(row["unigrams_ques1"] ).union(set(row["unigrams_ques2"]))),1)
 train_df["unigrams_ques1"] = train_df['question1'].apply(lambda x: get_unigrams(str(x)))
 train_df["unigrams_ques2"] = train_df['question2'].apply(lambda x: get_unigrams(str(x)))
 train_df["unigrams_common_count"] = train_df.apply(lambda row: get_common_unigrams(row),axis=1)
 train_df["unigrams_common_ratio"] = train_df.apply(lambda row: get_common_unigram_ratio(row), axis=1 )
X_train, y_train = prepare_data(df_train, building, weather_train)
 del df_train, weather_train
gc.collect()<create_dataframe>
ques = pd.concat([train_df[['question1', 'question2']], \
test_df[['question1', 'question2']]], axis=0 ).reset_index(drop='index')
 ques.shape
%%time
X_half_1 = X_train[:int(X_train.shape[0] / 2)]
X_half_2 = X_train[int(X_train.shape[0] / 2):]
y_half_1 = y_train[:int(X_train.shape[0] / 2)]
y_half_2 = y_train[int(X_train.shape[0] / 2):]
categorical_features = ["building_id", "meter", "primary_use", "hour", "weekday"]
d_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categorical_features, free_raw_data=False)
 d_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categorical_features, free_raw_data=False)
 watchlist_1 = [d_half_1, d_half_2]
watchlist_2 = [d_half_2, d_half_1]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 40,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 2,
"metric": "rmse",
"num_threads": 2
}
print("Building model with first half and validating on second half:")
 model_half_1 = lgb.train(params, train_set=d_half_1, num_boost_round=1000, valid_sets=watchlist_1, verbose_eval=200, early_stopping_rounds=200)
 print("Building model with second half and validating on first half:")
 model_half_2 = lgb.train(params, train_set=d_half_2, num_boost_round=1000, valid_sets=watchlist_2, verbose_eval=200, early_stopping_rounds=200 )<predict_on_test>
q_dict = defaultdict(set)
 for i in range(ques.shape[0]):
q_dict[ques.question1[i]].add(ques.question2[i])
 q_dict[ques.question2[i]].add(ques.question1[i] )
%%time
pred = np.zeros(len(X_train))
pred1 = model_half_1.predict(X_half_2)
 pred2 = model_half_2.predict(X_half_1 )<compute_test_metric>
def q1_freq(row):
return(len(q_dict[row['question1']]))
def q2_freq(row):
return(len(q_dict[row['question2']]))
def q1_q2_intersect(row):
return(len(set(q_dict[row['question1']] ).intersection(set(q_dict[row['question2']]))))
train_df['q1_q2_intersect'] = train_df.apply(q1_q2_intersect, axis=1, raw=True)
 train_df['q1_freq'] = train_df.apply(q1_freq, axis=1, raw=True)
 train_df['q2_freq'] = train_df.apply(q2_freq, axis=1, raw=True )