Kaggle competition: WSDM - KKBox's Churn Prediction Challenge
<SOS> metric: LogLoss Kaggle data source: wsdm-kkboxs-churn-prediction-challenge<save_to_csv>
out_df = pd.DataFrame(test_pred/3)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("cz.csv", index=False )<feature_engineering>
%matplotlib inline
roc_curve, recall_score, classification_report, f1_score,
precision_recall_fscore_support )
train_df=pd.read_json('.. /input/train.json')
 test_df=pd.read_json('.. /input/test.json')
 train_df.head()
test_df["bathrooms"].loc[19671] = 1.5
test_df["bathrooms"].loc[22977] = 2.0
test_df["bathrooms"].loc[63719] = 2.0
ulimit = np.percentile(train_df.price.values, 99)
 train_df['price'].ix[train_df['price']>ulimit] = ulimit
ulimit = np.percentile(test_df.price.values, 99)
 test_df['price'].ix[test_df['price']>ulimit] = ulimit
train_df["logprice"] = np.log(train_df["price"])
 test_df["logprice"] = np.log(test_df["price"])
 train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["pos"] = train_df.longitude.round(3 ).astype(str)+ '_' + train_df.latitude.round(3 ).astype(str)
 test_df["pos"] = test_df.longitude.round(3 ).astype(str)+ '_' + test_df.latitude.round(3 ).astype(str)
 vals = train_df['pos'].value_counts()
dvals = vals.to_dict()
train_df["density"] = train_df['pos'].apply(lambda x: dvals.get(x, vals.min()))
test_df["density"] = test_df['pos'].apply(lambda x: dvals.get(x, vals.min()))
train_df["price_t"] =train_df["price"]/train_df["bedrooms"]
test_df["price_t"] = test_df["price"]/test_df["bedrooms"]
train_df["room_sum"] = train_df["bedrooms"]+train_df["bathrooms"]
test_df["room_sum"] = test_df["bedrooms"]+test_df["bathrooms"]
train_df['price_per_room'] = train_df['price']/train_df['room_sum']
test_df['price_per_room'] = test_df['price']/test_df['room_sum']
image_date = pd.read_csv('.. /input/listing_image_time.csv')
 image_date.columns = ["listing_id", "time_stamp"]
image_date.loc[80240,"time_stamp"] = 1478129766
image_date["img_date"] = pd.to_datetime(image_date["time_stamp"], unit="s")
 image_date["img_days_passed"] =(image_date["img_date"].max() - image_date["img_date"] ).astype("timedelta64[D]" ).astype(int)
 image_date["img_date_month"] = image_date["img_date"].dt.month
image_date["img_date_week"] = image_date["img_date"].dt.week
image_date["img_date_day"] = image_date["img_date"].dt.day
image_date["img_date_dayofweek"] = image_date["img_date"].dt.dayofweek
image_date["img_date_dayofyear"] = image_date["img_date"].dt.dayofyear
image_date["img_date_hour"] = image_date["img_date"].dt.hour
image_date["img_date_monthBeginMidEnd"] = image_date["img_date_day"].apply(lambda x: 1 if x<10 else 2 if x<20 else 3)
 train_df = pd.merge(train_df, image_date, on="listing_id", how="left")
 test_df = pd.merge(test_df, image_date, on="listing_id", how="left")
 features_to_use=["bathrooms", "bedrooms", "price_t","room_sum","latitude","longitude","num_photos","density","logprice","num_features","num_description_words","price_per_room","listing_id","img_date"]
print(train_df['price'].head())
<train_model>
train = pd.read_csv('.. /input/train.csv')
 train = pd.concat(( train, pd.read_csv('.. /input/train_v2.csv')) ,axis=0, ignore_index=True ).reset_index(drop=True)
 test = pd.read_csv('.. /input/sample_submission_v2.csv' )
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=321, num_rounds=818):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.03
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model 
<define_variables>
transactions = pd.read_csv('.. /input/transactions.csv', usecols=['msno'])
 transactions = pd.concat(( transactions, pd.read_csv('.. /input/transactions_v2.csv', usecols=['msno'])) , axis=0, ignore_index=True ).reset_index(drop=True)
 transactions = pd.DataFrame(transactions['msno'].value_counts().reset_index())
 transactions.columns = ['msno','trans_count']
train = pd.merge(train, transactions, how='left', on='msno')
 test = pd.merge(test, transactions, how='left', on='msno' )
a=[]
b=[]
c=[]
building_level={}
for j in train_df['manager_id'].values:
building_level[j]=[0,0,0]
for j in range(train_df.shape[0]):
temp=train_df.iloc[j]
if temp['interest_level']=='low':
building_level[temp['manager_id']][0]+=1
if temp['interest_level']=='medium':
building_level[temp['manager_id']][1]+=1
if temp['interest_level']=='high':
building_level[temp['manager_id']][2]+=1
for i in test_df['manager_id'].values:
if i not in building_level.keys() :
a.append(np.nan)
 b.append(np.nan)
 c.append(np.nan)
 else:
a.append(building_level[i][0]*1.0/sum(building_level[i]))
b.append(building_level[i][1]*1.0/sum(building_level[i]))
c.append(building_level[i][2]*1.0/sum(building_level[i]))
test_df['manager_level_low']=a
test_df['manager_level_medium']=b
test_df['manager_level_high']=c
features_to_use.append('manager_level_low')
 features_to_use.append('manager_level_medium')
 features_to_use.append('manager_level_high')
 categorical = ["display_address", "manager_id", "building_id", "street_address"] 
<categorify>
transactions = pd.read_csv('.. /input/transactions_v2.csv')
 transactions = transactions.sort_values(by=['transaction_date'], ascending=[False] ).reset_index(drop=True)
 transactions = transactions.drop_duplicates(subset=['msno'], keep='first')
 train = pd.merge(train, transactions, how='left', on='msno')
 test = pd.merge(test, transactions, how='left', on='msno')
 transactions=[]
for f in categorical:
	if train_df[f].dtype=='object':
		
		lbl = preprocessing.LabelEncoder()
		lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
		train_df[f] = lbl.transform(list(train_df[f].values))
		test_df[f] = lbl.transform(list(test_df[f].values))
		features_to_use.append(f)
<string_transform>
user_logs = pd.read_csv('.. /input/user_logs_v2.csv', usecols=['msno'])
 user_logs = pd.DataFrame(user_logs['msno'].value_counts().reset_index())
 user_logs.columns = ['msno','logs_count']
train = pd.merge(train, user_logs, how='left', on='msno')
 test = pd.merge(test, user_logs, how='left', on='msno')
 user_logs = [];
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x])) 
<categorify>
def transform_df(df):
df = pd.DataFrame(df)
 df = df.sort_values(by=['date'], ascending=[False])
 df = df.reset_index(drop=True)
 df = df.drop_duplicates(subset=['msno'], keep='first')
 return df
def transform_df2(df):
df = df.sort_values(by=['date'], ascending=[False])
 df = df.reset_index(drop=True)
 df = df.drop_duplicates(subset=['msno'], keep='first')
 return df
def factorize(df1, df2, column):
ps = df1[column].append(df2[column])
 factors = ps.factorize() [0]
df1[column] = factors[:len(df1)]
df2[column] = factors[len(df1):]
return df1, df2 
<categorify>
last_user_logs = []
last_user_logs.append(transform_df(pd.read_csv('.. /input/user_logs_v2.csv')))
 last_user_logs = pd.concat(last_user_logs, axis=0, ignore_index=True ).reset_index(drop=True)
 last_user_logs = transform_df2(last_user_logs)
 print('merging user logs features...')
 train = pd.merge(train, last_user_logs, how='left', on='msno')
 test = pd.merge(test, last_user_logs, how='left', on='msno')
 last_user_logs=[]
for col in('building_id', 'display_address', 'manager_id', 'street_address'):
train_df,test_df = factorize(train_df, test_df,col)
<categorify>
members = pd.read_csv('.. /input/members_v3.csv')
 train = pd.merge(train, members, how='left', on='msno')
 test = pd.merge(test, members, how='left', on='msno')
 members = []; print('members merge...' )
tfidf = CountVectorizer(stop_words='english', max_features=200)
 tfidf.fit(list(train_df['features'])+ list(test_df['features']))
tr_sparse = tfidf.transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"])
 print(te_sparse)
<concatenate>
gender = {'male':1, 'female':2}
train['gender'] = train['gender'].map(gender)
 test['gender'] = test['gender'].map(gender)
 train = train.fillna(0)
 test = test.fillna(0 )
train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr() 
<concatenate>
train = train.fillna(0)
 test = test.fillna(0)
 train.head()
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr() 
<prepare_x_and_y>
from keras.models import Sequential
from keras.callbacks import ReduceLROnPlateau
from keras.callbacks import LambdaCallback
from keras.layers.normalization import BatchNormalization
from keras.optimizers import SGD
from keras.constraints import maxnorm
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense, Lambda
from keras.layers.core import Dropout
from keras import regularizers
from keras.models import Model, load_model
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.layers import Embedding
from keras.layers import Conv1D, GlobalMaxPooling1D
from keras.datasets import imdb
from keras.models import Sequential
from keras.layers import Dense, Activation, MaxPooling1D
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape)
<save_to_csv>
cols = [c for c in train.columns if c not in ['is_churn','msno']]
X_train = StandardScaler().fit_transform(train[cols].as_matrix())
 y_train = train['is_churn'].as_matrix()
X_test = StandardScaler().fit_transform(test[cols].as_matrix() )
preds, model = runXGB(train_X, train_y, test_X, num_rounds=818)
 out_df = pd.DataFrame(preds)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("xgb_starter2.csv", index=False)
<set_options>
lsize = 128
model = Sequential()
model.add(Dense(lsize, input_dim=int(X_train.shape[1]),activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(rate=0.25))
model.add(Dense(int(lsize/2), activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(rate=0.25))
model.add(Dense(int(lsize/4),kernel_regularizer=regularizers.l2(0.1), activation='relu'))
model.add(Dropout(rate=0.1))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adadelta', metrics=['accuracy'])
 model.summary()
print("Tensorflow version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE<choose_model_class>
history = model.fit(X_train, y_train, epochs=10, batch_size=1026,
validation_split=0.2, verbose=1)
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 except ValueError:
strategy = tf.distribute.MirroredStrategy()
print("Number of accelerators: ", strategy.num_replicas_in_sync )<define_variables>
predictions = model.predict(X_test)
 test['is_churn'] = predictions.clip(0.+1e-15, 1-1e-15)
 test[['msno','is_churn']].to_csv('submission_NN.csv', index=False )