Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
<SOS> metric: AUC Kaggle data source: talkingdata-adtracking-fraud-detection-challenge<choose_model_class>
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
import lightgbm as lgb
import gc
pred_start = datetime.datetime.now()
elapsed =(pred_start - kernel_start ).total_seconds()
print(f'time before prediction: {elapsed} seconds')
 if PRED:
ckpt_no = 60
ckpt_manager, loaded_ckpt_no = load_ckpt(
 predictor=predictor, optimizer=None,
ckpt_path=CKPT_PRED_PATH, ckpt_no=ckpt_no, 
)
 if SUBMISSION:
assert loaded_ckpt_no > 0
pred_iter_manager_class = Test_Iter_Manager
pred_manager = Pred_Manager.create(
 config=config,
train_config=train_config,
pred_iter_manager_class=pred_iter_manager_class,
train_dt_path_or_obj=train_dt,
user_id_to_row_id_train_path=user_id_to_row_id_train_path,
unique_question_id_train_path=unique_question_id_train_path,
unique_lecture_id_train_path=unique_lecture_id_train_path,
question_history_at_training_end_path=question_history_at_training_end_path,
single_question_history_at_training_end_optimized_path=single_question_history_at_training_end_optimized_path,
valid_info_path=None,
max_train_buffer_size=30000,
probe=PROBE,
debug=False 
)
 run_pred(pred_manager, predictor, max_steps=None )<set_options>
cwd = os.getcwd()
train_path = '.. /input/'+'train.csv'
test_path = '.. /input/'+'test.csv'
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
 DATA_PATH = '.. /input/riiid-test-answer-prediction/'
MY_DATA_PATH = '.. /input/my_data/'
CACHE_PATH = '.. /input/lgb1215weights/'
if not os.path.exists(CACHE_PATH):
os.mkdir(CACHE_PATH)
 DEBUG = False
OFFLINE = False
if DEBUG:
MY_DATA_PATH = f'{MY_DATA_PATH}/debug/'
CACHE_PATH = f'{CACHE_PATH}/debug/'
if not os.path.exists(CACHE_PATH):
os.mkdir(CACHE_PATH)
 config_file = f'{CACHE_PATH}/config.pkl'
def save_pickle(dic, save_path):
with gzip.open(save_path, 'wb')as f:
pickle.dump(dic, f)
 def load_pickle(load_path):
with gzip.open(load_path, 'rb')as f:
message_dict = pickle.load(f)
 return message_dict
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024 ** 2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024 ** 2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(
 start_mem - end_mem)/ start_mem))
return df
questions_df = pd.read_csv(f'{DATA_PATH}/questions.csv')
 questions_df['content_bundle_same'] =(questions_df['question_id'] == questions_df['bundle_id'] ).astype(int)
 questions_df['tags_len'] = questions_df['tags'].apply(lambda x: 0 if str(x)== 'nan' else len(str(x ).split(' ')))
 questions_df['tags'] = questions_df['tags'].apply(lambda x: [] if str(x)== 'nan' else str(x ).split(' '))
questions_df['part_content_num'] = questions_df.groupby('part')['question_id'].transform('count')
 question_bundle_dict = dict(zip(questions_df.question_id.values, questions_df.bundle_id.values))
question_part_dict = dict(zip(questions_df.question_id.values, questions_df.part.values))
question_tags_dict = dict(zip(questions_df.question_id.values, questions_df.tags.values))
bundle_df = questions_df.groupby('bundle_id')['question_id'].unique()
bundle_df = bundle_df[bundle_df.apply(len)> 1]
bundle_mapping = {}
for id_list in bundle_df.values:
bid = id_list[0]
for qid in id_list:
bundle_mapping[qid] =(bid, len(id_list))
', bundle_mapping)
 lectures_df = pd.read_csv(f'{DATA_PATH}/lectures.csv')
 lecture_tag_dict = dict(zip(lectures_df.lecture_id.values, lectures_df.tag.values))
lecture_part_dict = dict(zip(lectures_df.lecture_id.values, lectures_df.part.values))
lecture_type_dict = dict(zip(lectures_df.lecture_id.values, lectures_df.type_of.values))
if OFFLINE:
feld_needed = ['row_id', 'timestamp', 'user_id', 'content_id', 'content_type_id', 'task_container_id',
'user_answer', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation']
train = pd.read_pickle(f'{MY_DATA_PATH}/train.pickle')[feld_needed]
valid = pd.read_pickle(f'{MY_DATA_PATH}/valid.pickle')[feld_needed]
train['prior_question_elapsed_time'] //= 1000
train['timestamp'] /= 1000
valid['prior_question_elapsed_time'] //= 1000
valid['timestamp'] /= 1000
print(f'Train|Valid: {len(train)}|{len(valid)}')

ques_train = train.loc[train.content_type_id == False, ['row_id', 'content_id', 'answered_correctly']].reset_index(
 drop=True)
 ques_valid = valid.loc[valid.content_type_id == False, ['row_id', 'content_id', 'answered_correctly']].reset_index(
 drop=True)
 ques_train = reduce_mem_usage(ques_train, verbose=True)
 ques_valid = reduce_mem_usage(ques_valid, verbose=True)
 train['prior_question_had_explanation'] = train['prior_question_had_explanation'].fillna(False ).astype('int8')
 valid['prior_question_had_explanation'] = valid['prior_question_had_explanation'].fillna(False ).astype('int8')
 prior_question_elapsed_time_mean = train.loc[train.content_type_id == False].prior_question_elapsed_time.dropna().values.mean()
train['prior_question_elapsed_time'] = train['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean ).astype(np.int32)
 valid['prior_question_elapsed_time'] = valid['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean ).astype(np.int32)
 train = reduce_mem_usage(train, verbose=True)
 valid = reduce_mem_usage(valid, verbose=True)
 save_pickle(prior_question_elapsed_time_mean, f'{CACHE_PATH}/prior_question_elapsed_time_mean.pkl')
 config_dict = {}
else:
config_dict = load_pickle(config_file)
 prior_question_elapsed_time_mean = load_pickle(f'{CACHE_PATH}/prior_question_elapsed_time_mean.pkl')
 def make_content_feat(df, type):
df = df.loc[df.content_type_id == False].reset_index(drop=True)
 file_name = f'content_feat_{type}.pkl'
feat_df = df.groupby('content_id', as_index=False)['answered_correctly'].mean().\
rename(columns={'answered_correctly': 'content_target_mean'})
 content_cnt = df.groupby('content_id')['user_id'].count()
feat_df['content_cnt'] = content_cnt.reindex(feat_df.content_id.values ).values
save_pickle(feat_df, save_path=f'{CACHE_PATH}/{file_name}')
 feat_df = reduce_mem_usage(feat_df, verbose=True)
 return feat_df
if OFFLINE:
content_feat = make_content_feat(df=train.copy(deep=True), type='train')
 content_feat_test = make_content_feat(df=pd.concat([train, valid]), type='test')
 print('content_feat:
', content_feat.head())
 else:
content_feat_test = load_pickle(f'{CACHE_PATH}/content_feat_test.pkl')
 content_feat_test = reduce_mem_usage(content_feat_test, verbose=True)
 content_target_mean_dict = dict(zip(content_feat_test.content_id.values,
content_feat_test.content_target_mean.values))
content_feat_cols = [col for col in content_feat_test if col != 'content_id']
def make_part_mean_dict(df):
df = df.loc[df.content_type_id == False].reset_index(drop=True)
 df = df.merge(questions_df[['question_id', 'part']], left_on='content_id', right_on='question_id', how='left')
 feat_df = df.groupby('part', as_index=False)['answered_correctly'].mean().\
rename(columns={'answered_correctly': 'part_target_mean'})
 return dict(zip(feat_df.part.values, feat_df.part_target_mean.values))
if OFFLINE:
part_target_mean_dict = make_part_mean_dict(df=pd.concat([train[['content_id', 'content_type_id', 'answered_correctly']],
valid[['content_id', 'content_type_id', 'answered_correctly']]]))
save_pickle(part_target_mean_dict, f'{CACHE_PATH}/part_target_mean_dict.pkl')
 else:
part_target_mean_dict = load_pickle(f'{CACHE_PATH}/part_target_mean_dict.pkl')
 questions_df['part_target_mean'] = questions_df['part'].apply(lambda x: part_target_mean_dict[x])
 static_feat_cols = ['part', 'prior_question_elapsed_time', 'content_id']
def get_stat_feat(df, feat_cols):
df = df.loc[df.content_type_id == False].reset_index(drop=True)
 df = df.merge(questions_df[['question_id', 'part']], left_on='content_id', right_on='question_id', how='left')
 return df[feat_cols]
if OFFLINE:
state_feat_train = get_stat_feat(df=train.copy(deep=True), feat_cols=static_feat_cols)
 state_feat_valid = get_stat_feat(df=valid.copy(deep=True), feat_cols=static_feat_cols)
 state_feat_train = reduce_mem_usage(state_feat_train, verbose=True)
 state_feat_valid = reduce_mem_usage(state_feat_valid, verbose=True)
 window_size = 25
if OFFLINE:
user_cnt_dict = defaultdict(int)
 user_pos_cnt_dict = defaultdict(int)
 user_part_cnt_dict = defaultdict(int)
 user_part_pos_cnt_dict = defaultdict(int)
 user_content_cnt_dict = defaultdict(int)
 user_content_pos_cnt_dict = defaultdict(int)
 user_content_redo_cnt_dict = defaultdict(int)
 user_content_mean_sum_dict = defaultdict(int)
 user_consecutive_pos_cnt_dict = defaultdict(int)
 user_target_win25_dict = defaultdict(list)
 user_content_mean_win10_dict = defaultdict(list)
 user_explanation_cnt_dict = defaultdict(int)
 user_explanation_pos_cnt_dict = defaultdict(int)
 user_elapse_time_sum_dict = defaultdict(int)
 user_elapse_time_win10_dict = defaultdict(list)
 user_last_timestamp_dict = defaultdict(int)
 user_last_task_dict = defaultdict(int)
 user_content_win5_dict = defaultdict(list)
 user_part_win10_dict = defaultdict(list)
 bundle_state_dict = defaultdict(list)
 user_cum_time_dict = defaultdict(int)
 user_timespan_win10_dict = defaultdict(list)
 user_tags_cnt_dict = defaultdict(int)
 user_tags_pos_cnt_dict = defaultdict(int)
 user_continue_quest_cnt_dict = defaultdict(int)
 else:
user_content_feat_df = pd.read_pickle(f'{CACHE_PATH}/user_content_feat.pkl')
 user_content_feat_df = reduce_mem_usage(user_content_feat_df)
 user_content_cnt_dict = defaultdict(int)
 user_content_pos_cnt_dict = defaultdict(int)
 user_cnt_dict = load_pickle(f'{CACHE_PATH}/user_cnt_dict.pkl')
 user_pos_cnt_dict = load_pickle(f'{CACHE_PATH}/user_pos_cnt_dict.pkl')
 user_part_cnt_dict = load_pickle(f'{CACHE_PATH}/user_part_cnt_dict.pkl')
 user_part_pos_cnt_dict = load_pickle(f'{CACHE_PATH}/user_part_pos_cnt_dict.pkl')
 user_content_redo_cnt_dict = load_pickle(f'{CACHE_PATH}/user_content_redo_cnt_dict.pkl')
 user_content_mean_sum_dict = load_pickle(f'{CACHE_PATH}/user_content_mean_sum_dict.pkl')
 user_consecutive_pos_cnt_dict = load_pickle(f'{CACHE_PATH}/user_consecutive_pos_cnt_dict.pkl')
 user_target_win25_dict = load_pickle(f'{CACHE_PATH}/user_target_win25_dict.pkl')
 user_content_mean_win10_dict = load_pickle(f'{CACHE_PATH}/user_content_mean_win10_dict.pkl')
 user_explanation_cnt_dict = load_pickle(f'{CACHE_PATH}/user_explanation_cnt_dict.pkl')
 user_explanation_pos_cnt_dict = load_pickle(f'{CACHE_PATH}/user_explanation_pos_cnt_dict.pkl')
 user_elapse_time_sum_dict = load_pickle(f'{CACHE_PATH}/user_elapse_time_sum_dict.pkl')
 user_elapse_time_win10_dict = load_pickle(f'{CACHE_PATH}/user_elapse_time_win10_dict.pkl')
 user_last_timestamp_dict = load_pickle(f'{CACHE_PATH}/user_last_timestamp_dict.pkl')
 user_last_task_dict = load_pickle(f'{CACHE_PATH}/user_last_task_dict.pkl')
 user_content_win5_dict = load_pickle(f'{CACHE_PATH}/user_content_win5_dict.pkl')
 user_part_win10_dict = load_pickle(f'{CACHE_PATH}/user_part_win10_dict.pkl')
 bundle_state_dict = load_pickle(f'{CACHE_PATH}/bundle_state_dict.pkl')
 user_cum_time_dict = load_pickle(f'{CACHE_PATH}/user_cum_time_dict.pkl')
 user_timespan_win10_dict = load_pickle(f'{CACHE_PATH}/user_timespan_win10_dict.pkl')
 user_tags_cnt_dict = load_pickle(f'{CACHE_PATH}/user_tags_cnt_dict.pkl')
 user_tags_pos_cnt_dict = load_pickle(f'{CACHE_PATH}/user_tags_pos_cnt_dict.pkl')
 user_continue_quest_cnt_dict = load_pickle(f'{CACHE_PATH}/user_continue_quest_cnt_dict.pkl')
 used_cols = ['user_id', 'content_id', 'task_container_id', 'answered_correctly', 'prior_question_elapsed_time',
'prior_question_had_explanation', 'content_type_id', 'timestamp']
def make_user_loop_features(df, content_target_mean_dict,
user_cnt_dict, user_pos_cnt_dict,
user_part_cnt_dict, user_part_pos_cnt_dict,
user_content_cnt_dict, user_content_pos_cnt_dict, user_content_redo_cnt_dict, user_content_mean_sum_dict,
user_consecutive_pos_cnt_dict, user_target_win25_dict, user_content_mean_win10_dict,
user_explanation_cnt_dict, user_explanation_pos_cnt_dict, user_elapse_time_sum_dict, user_elapse_time_win10_dict,
user_last_timestamp_dict, user_last_task_dict,
user_content_win5_dict, user_part_win10_dict,
bundle_state_dict, user_cum_time_dict, user_timespan_win10_dict,
user_tags_cnt_dict, user_tags_pos_cnt_dict,
user_continue_quest_cnt_dict,
update=True, isTrain=True):
sample_num = len(df.loc[df.content_type_id == False])
 user_cnt_npy = np.zeros(sample_num)
 user_pos_cnt_npy = np.zeros(sample_num)
 user_part_cnt_npy = np.zeros(sample_num)
 user_part_pos_cnt_npy = np.zeros(sample_num)
 user_content_cnt_npy = np.zeros(sample_num)
 user_content_pos_cnt_npy = np.zeros(sample_num)
 user_content_redo_cnt_npy = np.zeros(sample_num)
 user_content_mean_mean_npy = np.zeros(sample_num)
 user_consecutive_pos_cnt_npy = np.zeros(sample_num)
 user_pos_cnt_win25_npy = np.zeros(sample_num)
 user_content_mean_win10_npy = np.zeros(sample_num)
 user_explanation_cnt_npy = np.zeros(sample_num)
 user_explanation_pos_cnt_npy = np.zeros(sample_num)
 user_elapse_time_mean_npy = np.zeros(sample_num)
 user_elapse_time_mean_win10_npy = np.zeros(sample_num)
 user_last_timespan_npy = np.zeros(sample_num)
 user_last_task_diff_npy = np.zeros(sample_num)
 user_content_appear_in_win5_npy = np.zeros(sample_num)
 user_part_cnt_in_win10_npy = np.zeros(sample_num)
 user_cum_time_npy = np.zeros(sample_num)
 user_timespan_win10_mean_npy = np.zeros(sample_num)
 user_tags_cnt_mean_npy = np.zeros(sample_num)
 user_tags_pos_rate_npy = np.zeros(sample_num)
 user_continue_quest_cnt_npy = np.zeros(sample_num)
 if update:
tk0 = tqdm(df[used_cols].values)
 else:
tk0 = df[used_cols].values
idx = 0
for(_user_id, _content_id, _task_container_id, _answered_correctly, _prior_question_elapsed_time,
_prior_question_had_explanation, _content_type_id, _timestamp)in tk0:
if _content_id in content_target_mean_dict:
_content_target_mean = content_target_mean_dict[_content_id]
else:
_content_target_mean = 0
if _content_type_id == False:
_bundle_id = question_bundle_dict[_content_id]
_part = question_part_dict[_content_id]
_tags = question_tags_dict[_content_id]
user_cnt_npy[idx] = user_cnt_dict[_user_id]
user_cnt_dict[_user_id] += 1
user_pos_cnt_npy[idx] = user_pos_cnt_dict[_user_id]
if update:
user_pos_cnt_dict[_user_id] += _answered_correctly
_user_part = str(_user_id)+ '_' + str(_part)
 user_part_cnt_npy[idx] = user_part_cnt_dict[_user_part]
user_part_cnt_dict[_user_part] += 1
user_part_pos_cnt_npy[idx] = user_part_pos_cnt_dict[_user_part]
if update:
user_part_pos_cnt_dict[_user_part] += _answered_correctly
_user_content = np.int64(_user_id)* 10_0000 + _content_id
if isTrain:
user_content_cnt_npy[idx] = user_content_cnt_dict[_user_content]
user_content_cnt_dict[_user_content] += 1
else:
if _user_content in user_content_feat_df.index:
user_content_cnt_npy[idx] = user_content_feat_df.loc[_user_content]['user_content_cnt'] + user_content_cnt_dict[_user_content]
else:
user_content_cnt_npy[idx] = user_content_cnt_dict[_user_content]
user_content_cnt_dict[_user_content] += 1
if isTrain:
user_content_pos_cnt_npy[idx] = user_content_pos_cnt_dict[_user_content]
if update:
user_content_pos_cnt_dict[_user_content] += _answered_correctly
else:
if _user_content in user_content_feat_df.index:
user_content_pos_cnt_npy[idx] = user_content_feat_df.loc[_user_content]['user_content_pos_cnt'] + user_content_pos_cnt_dict[_user_content]
else:
user_content_pos_cnt_npy[idx] = user_content_pos_cnt_dict[_user_content]
user_content_redo_cnt_npy[idx] = user_content_redo_cnt_dict[_user_id]
if isTrain:
if user_content_cnt_dict[_user_content] > 0:
user_content_redo_cnt_dict[_user_id] += 1
else:
if _user_content in user_content_feat_df.index:
if user_content_feat_df.loc[_user_content]['user_content_cnt'] > 0:
user_content_redo_cnt_dict[_user_id] += 1
else:
if user_content_cnt_dict[_user_content] > 0:
user_content_redo_cnt_dict[_user_id] += 1
user_content_mean_mean_npy[idx] = user_content_mean_sum_dict[_user_id]
user_content_mean_sum_dict[_user_id] += _content_target_mean
user_consecutive_pos_cnt_npy[idx] = user_consecutive_pos_cnt_dict[_user_id]
if update:
if _answered_correctly:
user_consecutive_pos_cnt_dict[_user_id] += 1
else:
user_consecutive_pos_cnt_dict[_user_id] = 0
user_pos_cnt_win25_npy[idx] = sum(user_target_win25_dict[_user_id])
 if update:
user_target_win25_dict[_user_id].append(_answered_correctly)
 if len(user_target_win25_dict[_user_id])> 25:
tmp = user_target_win25_dict[_user_id]
user_target_win25_dict[_user_id] = tmp[-25:]
div_num = len(user_content_mean_win10_dict[_user_id])
 if div_num > 0:
user_content_mean_win10_npy[idx] = sum(user_content_mean_win10_dict[_user_id])/ div_num
user_content_mean_win10_dict[_user_id].append(_content_target_mean)
 if len(user_content_mean_win10_dict[_user_id])> 10:
tmp = user_content_mean_win10_dict[_user_id]
user_content_mean_win10_dict[_user_id] = tmp[-10:]
user_explanation_cnt_npy[idx] = user_explanation_cnt_dict[_user_id]
user_explanation_cnt_dict[_user_id] += _prior_question_had_explanation
user_explanation_pos_cnt_npy[idx] =user_explanation_pos_cnt_dict[_user_id]
if update:
if _answered_correctly:
user_explanation_pos_cnt_dict[_user_id] += _prior_question_had_explanation
user_elapse_time_mean_npy[idx] = user_elapse_time_sum_dict[_user_id]
user_elapse_time_sum_dict[_user_id] += _prior_question_elapsed_time
div_num = len(user_elapse_time_win10_dict[_user_id])
 if div_num > 0:
user_elapse_time_mean_win10_npy[idx] = sum(user_elapse_time_win10_dict[_user_id])/ div_num
user_elapse_time_win10_dict[_user_id].append(_prior_question_elapsed_time)
 if len(user_elapse_time_win10_dict[_user_id])> 10:
tmp = user_elapse_time_win10_dict[_user_id]
user_elapse_time_win10_dict[_user_id] = tmp[-10:]
time_diff = _timestamp - user_last_timestamp_dict[_user_id]
if _content_id in bundle_mapping:
if len(bundle_state_dict[_user_id])> 1:
previous_bundle_id, previous_diff = bundle_state_dict[_user_id]
if bundle_mapping[_content_id][0] == previous_bundle_id:
time_diff = previous_diff
else:
time_diff = time_diff // bundle_mapping[_content_id][1]
bundle_state_dict[_user_id] = [bundle_mapping[_content_id][0], time_diff]
else:
time_diff = time_diff // bundle_mapping[_content_id][1]
bundle_state_dict[_user_id] = [bundle_mapping[_content_id][0], time_diff]
else:
bundle_state_dict[_user_id] = []
user_last_timespan_npy[idx] = time_diff
user_last_timestamp_dict[_user_id] = _timestamp
user_last_task_diff_npy[idx] = _task_container_id - user_last_task_dict[_user_id]
user_last_task_dict[_user_id] = _task_container_id
user_content_appear_in_win5_npy[idx] = _content_id in user_content_win5_dict[_user_id]
user_content_win5_dict[_user_id].append(_content_id)
 if len(user_content_win5_dict[_user_id])> 5:
del user_content_win5_dict[_user_id][0]
user_part_cnt_in_win10_npy[idx] = user_part_win10_dict[_user_id].count(_part)
 user_part_win10_dict[_user_id].append(_part)
 if len(user_part_win10_dict[_user_id])> 10:
del user_part_win10_dict[_user_id][0]
div_num = 0
for _tag in _tags:
_user_tag = str(_user_id)+ '_' + str(_tag)
 user_tags_cnt_mean_npy[idx] += user_tags_cnt_dict[_user_tag]
if user_tags_cnt_dict[_user_tag] > 0:
div_num += 1
if user_tags_cnt_dict[_user_tag] > 1:
_pos_rate = user_tags_pos_cnt_dict[_user_tag] / user_tags_cnt_dict[_user_tag]
user_tags_pos_rate_npy[idx] += _pos_rate
user_tags_cnt_dict[_user_tag] += 1
if update:
user_tags_pos_cnt_dict[_user_tag] += _answered_correctly
if div_num > 0:
user_tags_cnt_mean_npy[idx] //= div_num
user_tags_pos_rate_npy[idx] /= div_num
if time_diff <= 5 * 60:
user_cum_time_dict[_user_id] += time_diff
user_cum_time_npy[idx] = user_cum_time_dict[_user_id]
if time_diff <= 5 * 60:
user_timespan_win10_dict[_user_id].append(time_diff)
 if len(user_timespan_win10_dict[_user_id])> 10:
del user_timespan_win10_dict[_user_id][0]
div_num = len(user_timespan_win10_dict[_user_id])
 if div_num > 0:
user_timespan_win10_mean_npy[idx] = sum(user_timespan_win10_dict[_user_id])/ div_num
user_continue_quest_cnt_npy[idx] = user_continue_quest_cnt_dict[_user_id]
user_continue_quest_cnt_dict[_user_id] += 1
idx += 1
else:
_tag = lecture_tag_dict[_content_id]
_part = lecture_part_dict[_content_id]
_type_of = lecture_type_dict[_content_id]
_user_part = str(_user_id)+ '_' + str(_part)
 user_continue_quest_cnt_dict[_user_id] = 0
feats_df = pd.DataFrame({
'user_cnt': user_cnt_npy,
'user_pos_cnt': user_pos_cnt_npy,
'user_part_cnt': user_part_cnt_npy,
'user_part_pos_cnt': user_part_pos_cnt_npy,
'user_content_cnt': user_content_cnt_npy,
'user_content_pos_cnt': user_content_pos_cnt_npy,
'user_content_redo_cnt': user_content_redo_cnt_npy,
'user_content_mean_mean': user_content_mean_mean_npy,
'user_consecutive_pos_cnt': user_consecutive_pos_cnt_npy,
'user_pos_cnt_win25': user_pos_cnt_win25_npy,
'user_content_mean_win10': user_content_mean_win10_npy,
'user_explanation_cnt': user_explanation_cnt_npy,
'user_explanation_pos_cnt': user_explanation_pos_cnt_npy,
'user_elapse_time_mean': user_elapse_time_mean_npy,
'user_elapse_time_mean_win10': user_elapse_time_mean_win10_npy,
'user_last_timespan': user_last_timespan_npy,
'user_last_task_diff': user_last_task_diff_npy,
'user_content_appear_in_win5': user_content_appear_in_win5_npy,
'user_part_cnt_in_win10': user_part_cnt_in_win10_npy,
'user_cum_time': user_cum_time_npy,
'user_timespan_win10_mean': user_timespan_win10_mean_npy,
'user_tags_cnt_mean': user_tags_cnt_mean_npy,
'user_tags_pos_rate': user_tags_pos_rate_npy,
'user_continue_quest_cnt': user_continue_quest_cnt_npy,
})
 feats_df['user_target_mean'] = feats_df['user_pos_cnt'] / feats_df['user_cnt']
feats_df['user_part_target_mean'] = feats_df['user_part_pos_cnt'] / feats_df['user_part_cnt']
feats_df['user_content_target_mean'] = feats_df['user_content_pos_cnt'] / feats_df['user_content_cnt']
feats_df['user_content_mean_mean'] /= feats_df['user_cnt']
feats_df['user_explanation_mean'] = feats_df['user_explanation_cnt'] / feats_df['user_cnt']
feats_df['user_explanation_rate'] = feats_df['user_explanation_pos_cnt'] / feats_df['user_explanation_cnt']
feats_df['user_elapse_time_mean'] /= feats_df['user_cnt']
feats_df = feats_df.drop(['user_explanation_cnt', 'user_explanation_pos_cnt'], axis=1)
 return feats_df
def update_user_feats(df, user_content_pos_cnt_dict,
user_pos_cnt_dict, user_part_pos_cnt_dict, user_consecutive_pos_cnt_dict, user_target_win25_dict,
user_explanation_pos_cnt_dict, user_tags_pos_cnt_dict 
):
for(_user_id, _content_id, _task_container_id, _answered_correctly, _prior_question_elapsed_time,
_prior_question_had_explanation, _content_type_id, _timestamp)in df[used_cols].values:
if _content_type_id == False:
_bundle_id = question_bundle_dict[_content_id]
_part = question_part_dict[_content_id]
_tags = question_tags_dict[_content_id]
user_pos_cnt_dict[_user_id] += _answered_correctly
_user_part = str(_user_id)+ '_' + str(_part)
 user_part_pos_cnt_dict[_user_part] += _answered_correctly
_user_content = np.int64(_user_id)* 10_0000 + _content_id
user_content_pos_cnt_dict[_user_content] += _answered_correctly
if _answered_correctly:
user_consecutive_pos_cnt_dict[_user_id] += 1
else:
user_consecutive_pos_cnt_dict[_user_id] = 0
user_target_win25_dict[_user_id].append(_answered_correctly)
 if len(user_target_win25_dict[_user_id])> 25:
tmp = user_target_win25_dict[_user_id]
user_target_win25_dict[_user_id] = tmp[-25:]
if _answered_correctly:
user_explanation_pos_cnt_dict[_user_id] += _prior_question_had_explanation
for _tag in _tags:
_user_tag = str(_user_id)+ '_' + str(_tag)
 user_tags_pos_cnt_dict[_user_tag] += _answered_correctly
else:
_tag = lecture_tag_dict[_content_id]
_part = lecture_part_dict[_content_id]
_type_of = lecture_type_dict[_content_id]
if OFFLINE:
user_feat_train = make_user_loop_features(
 df=train, content_target_mean_dict=content_target_mean_dict,
user_cnt_dict=user_cnt_dict,
user_pos_cnt_dict=user_pos_cnt_dict,
user_part_cnt_dict=user_part_cnt_dict,
user_part_pos_cnt_dict=user_part_pos_cnt_dict,
user_content_cnt_dict=user_content_cnt_dict,
user_content_pos_cnt_dict=user_content_pos_cnt_dict,
user_content_redo_cnt_dict=user_content_redo_cnt_dict,
user_content_mean_sum_dict=user_content_mean_sum_dict,
user_consecutive_pos_cnt_dict=user_consecutive_pos_cnt_dict,
user_target_win25_dict=user_target_win25_dict,
user_content_mean_win10_dict=user_content_mean_win10_dict,
user_explanation_cnt_dict=user_explanation_cnt_dict,
user_explanation_pos_cnt_dict=user_explanation_pos_cnt_dict,
user_elapse_time_sum_dict=user_elapse_time_sum_dict,
user_elapse_time_win10_dict=user_elapse_time_win10_dict,
user_last_timestamp_dict=user_last_timestamp_dict,
user_last_task_dict=user_last_task_dict,
user_content_win5_dict=user_content_win5_dict,
user_part_win10_dict=user_part_win10_dict,
bundle_state_dict=bundle_state_dict,
user_cum_time_dict=user_cum_time_dict,
user_timespan_win10_dict=user_timespan_win10_dict,
user_tags_cnt_dict=user_tags_cnt_dict,
user_tags_pos_cnt_dict=user_tags_pos_cnt_dict,
user_continue_quest_cnt_dict=user_continue_quest_cnt_dict,
update=True, isTrain=True 
)
 user_feat_train = reduce_mem_usage(user_feat_train, verbose=True)
 user_feat_valid = make_user_loop_features(
 df=valid, content_target_mean_dict=content_target_mean_dict,
user_cnt_dict=user_cnt_dict,
user_pos_cnt_dict=user_pos_cnt_dict,
user_part_cnt_dict=user_part_cnt_dict,
user_part_pos_cnt_dict=user_part_pos_cnt_dict,
user_content_cnt_dict=user_content_cnt_dict,
user_content_pos_cnt_dict=user_content_pos_cnt_dict,
user_content_redo_cnt_dict=user_content_redo_cnt_dict,
user_content_mean_sum_dict=user_content_mean_sum_dict,
user_consecutive_pos_cnt_dict=user_consecutive_pos_cnt_dict,
user_target_win25_dict=user_target_win25_dict,
user_content_mean_win10_dict=user_content_mean_win10_dict,
user_explanation_cnt_dict=user_explanation_cnt_dict,
user_explanation_pos_cnt_dict=user_explanation_pos_cnt_dict,
user_elapse_time_sum_dict=user_elapse_time_sum_dict,
user_elapse_time_win10_dict=user_elapse_time_win10_dict,
user_last_timestamp_dict=user_last_timestamp_dict,
user_last_task_dict=user_last_task_dict,
user_content_win5_dict=user_content_win5_dict,
user_part_win10_dict=user_part_win10_dict,
bundle_state_dict=bundle_state_dict,
user_cum_time_dict=user_cum_time_dict,
user_timespan_win10_dict=user_timespan_win10_dict,
user_tags_cnt_dict=user_tags_cnt_dict,
user_tags_pos_cnt_dict=user_tags_pos_cnt_dict,
user_continue_quest_cnt_dict=user_continue_quest_cnt_dict,
update=True, isTrain=True 
)
 user_feat_valid = reduce_mem_usage(user_feat_valid, verbose=True)
 del train, valid
gc.collect()
user_feat_cols = user_feat_train.columns.values.tolist()
config_dict['user_feat_cols'] = user_feat_cols
user_feat_cols = config_dict['user_feat_cols']
def make_content_feat2(id_df, df, type):
df['content_id'] = id_df['content_id'].values
file_name = f'content_feat2_{type}.pkl'
feat_df = df.groupby('content_id', as_index=False)['user_last_timespan'].median().\
rename(columns={'answered_correctly': 'content_timespan_median'})
 save_pickle(feat_df, save_path=f'{CACHE_PATH}/{file_name}')
 feat_df = reduce_mem_usage(feat_df, verbose=True)
 return feat_df
if OFFLINE:
content_feat2 = make_content_feat2(id_df=ques_train, df=user_feat_train.copy(deep=True), type='train')
 content_feat2_test = make_content_feat2(id_df=pd.concat([ques_train, ques_valid]), df=pd.concat([user_feat_train, user_feat_valid]), type='test')
 print('content_feat2:
', content_feat2.head())
 else:
content_feat2_test = load_pickle(f'{CACHE_PATH}/content_feat2_test.pkl')
 content_feat2_test = reduce_mem_usage(content_feat2_test, verbose=True)
 content_feat2_cols = [col for col in content_feat2_test if col != 'content_id']
def merge_features(df, static_feat, content_feat, content_feat2, user_feat):
feat_df = static_feat.copy(deep=True)
 feat_df[content_feat_cols] = content_feat.set_index('content_id' ).reindex(df['content_id'].values ).values
feat_df[content_feat2_cols] = content_feat2.set_index('content_id' ).reindex(df['content_id'].values ).values
for col in user_feat_cols:
feat_df[col] = user_feat[col].values
return feat_df
if OFFLINE:
train_feat = merge_features(df=ques_train,
static_feat=state_feat_train,
content_feat=content_feat,
content_feat2=content_feat2,
user_feat=user_feat_train)
 del state_feat_train, user_feat_train
gc.collect()
train_feat = reduce_mem_usage(train_feat, verbose=True)
 valid_feat = merge_features(df=ques_valid,
static_feat=state_feat_valid,
content_feat=content_feat,
content_feat2=content_feat2,
user_feat=user_feat_valid)
 del state_feat_valid, user_feat_valid, content_feat
gc.collect()
valid_feat = reduce_mem_usage(valid_feat, verbose=True)
 feat_cols = static_feat_cols + content_feat_cols + content_feat2_cols + user_feat_cols
print(f'Feat Nums: {len(feat_cols)}')
 TARGET = 'answered_correctly'
if OFFLINE:
train_labels = ques_train[TARGET]
valid_labels = ques_valid[TARGET]
del ques_train, ques_valid
gc.collect()
lgb_train = lgb.Dataset(train_feat[feat_cols], train_labels, categorical_feature=['content_id'])
 lgb_valid = lgb.Dataset(valid_feat[feat_cols], valid_labels, categorical_feature=['content_id'])
 del train_feat, train_labels
gc.collect()
params = {
'objective': 'binary',
'seed': 28,
'num_leaves': 256,
'max_bin': 1024,
'feature_fraction': 0.6,
'max_depth': 8,
'verbose': -1,
'cat_l2': 30,
'cat_smooth': 20,
'num_threads': 20,
}
feature_importance_df = pd.DataFrame()
feature_importance_df["feature"] = feat_cols
model = lgb.train(
 params, lgb_train,
valid_sets=lgb_valid,
verbose_eval=50,
num_boost_round=10000,
early_stopping_rounds=50 
)
 model.save_model(f'{CACHE_PATH}/model.txt')
 feature_importance_df['importance'] = model.feature_importance()
feature_importance_df = feature_importance_df.sort_values('importance', ascending=False)
 feature_importance_df.to_csv(f'{CACHE_PATH}/feature_importance.csv', index=False)
 valid_score = roc_auc_score(valid_labels, model.predict(valid_feat[feat_cols]))
print(f'auc: {valid_score:.4f}')
 user_content_feat_df = pd.DataFrame()
user_content_feat_df['user_content_id'] = user_content_cnt_dict.keys()
user_content_feat_df['user_content_cnt'] = user_content_cnt_dict.values()
user_content_pos_cnt_feat_buff = pd.DataFrame()
user_content_pos_cnt_feat_buff['user_content_id'] = user_content_pos_cnt_dict.keys()
user_content_pos_cnt_feat_buff['user_content_pos_cnt'] = user_content_pos_cnt_dict.values()
user_content_feat_df = user_content_feat_df.merge(user_content_pos_cnt_feat_buff, on='user_content_id', how='left')
 del user_content_pos_cnt_feat_buff
user_content_feat_df = user_content_feat_df.set_index('user_content_id')
 user_content_feat_df.to_pickle(f'{CACHE_PATH}/user_content_feat.pkl')
 save_pickle(config_dict, config_file)
 save_pickle(user_cnt_dict, f'{CACHE_PATH}/user_cnt_dict.pkl')
 save_pickle(user_pos_cnt_dict, f'{CACHE_PATH}/user_pos_cnt_dict.pkl')
 save_pickle(user_part_cnt_dict, f'{CACHE_PATH}/user_part_cnt_dict.pkl')
 save_pickle(user_part_pos_cnt_dict, f'{CACHE_PATH}/user_part_pos_cnt_dict.pkl')
 save_pickle(user_content_redo_cnt_dict, f'{CACHE_PATH}/user_content_redo_cnt_dict.pkl')
 save_pickle(user_content_mean_sum_dict, f'{CACHE_PATH}/user_content_mean_sum_dict.pkl')
 save_pickle(user_consecutive_pos_cnt_dict, f'{CACHE_PATH}/user_consecutive_pos_cnt_dict.pkl')
 save_pickle(user_target_win25_dict, f'{CACHE_PATH}/user_target_win25_dict.pkl')
 save_pickle(user_content_mean_win10_dict, f'{CACHE_PATH}/user_content_mean_win10_dict.pkl')
 save_pickle(user_explanation_cnt_dict, f'{CACHE_PATH}/user_explanation_cnt_dict.pkl')
 save_pickle(user_explanation_pos_cnt_dict, f'{CACHE_PATH}/user_explanation_pos_cnt_dict.pkl')
 save_pickle(user_elapse_time_sum_dict, f'{CACHE_PATH}/user_elapse_time_sum_dict.pkl')
 save_pickle(user_elapse_time_win10_dict, f'{CACHE_PATH}/user_elapse_time_win10_dict.pkl')
 save_pickle(user_last_timestamp_dict, f'{CACHE_PATH}/user_last_timestamp_dict.pkl')
 save_pickle(user_last_task_dict, f'{CACHE_PATH}/user_last_task_dict.pkl')
 save_pickle(user_content_win5_dict, f'{CACHE_PATH}/user_content_win5_dict.pkl')
 save_pickle(user_part_win10_dict, f'{CACHE_PATH}/user_part_win10_dict.pkl')
 save_pickle(bundle_state_dict, f'{CACHE_PATH}/bundle_state_dict.pkl')
 save_pickle(user_cum_time_dict, f'{CACHE_PATH}/user_cum_time_dict.pkl')
 save_pickle(user_timespan_win10_dict, f'{CACHE_PATH}/user_timespan_win10_dict.pkl')
 save_pickle(user_tags_cnt_dict, f'{CACHE_PATH}/user_tags_cnt_dict.pkl')
 save_pickle(user_tags_pos_cnt_dict, f'{CACHE_PATH}/user_tags_pos_cnt_dict.pkl')
 save_pickle(user_continue_quest_cnt_dict, f'{CACHE_PATH}/user_continue_quest_cnt_dict.pkl')
 else:
model = lgb.Booster(model_file=f'{CACHE_PATH}/model.txt')
 print('load model finished')
 class Iter_Valid(object):
def __init__(self, df, max_user=1000):
df = df.reset_index(drop=True)
 self.df = df
self.user_answer = df['user_answer'].astype(str ).values
self.answered_correctly = df['answered_correctly'].astype(str ).values
df['prior_group_responses'] = "[]"
df['prior_group_answers_correct'] = "[]"
self.sample_df = df[df['content_type_id'] == 0][['row_id']]
self.sample_df['answered_correctly'] = 0
self.len = len(df)
 self.user_id = df.user_id.values
self.task_container_id = df.task_container_id.values
self.content_type_id = df.content_type_id.values
self.max_user = max_user
self.current = 0
self.pre_user_answer_list = []
self.pre_answered_correctly_list = []
def __iter__(self):
return self
def fix_df(self, user_answer_list, answered_correctly_list, pre_start):
df = self.df[pre_start:self.current].copy()
sample_df = self.sample_df[pre_start:self.current].copy()
df.loc[pre_start, 'prior_group_responses'] = '[' + ",".join(self.pre_user_answer_list)+ ']'
df.loc[pre_start, 'prior_group_answers_correct'] = '[' + ",".join(self.pre_answered_correctly_list)+ ']'
self.pre_user_answer_list = user_answer_list
self.pre_answered_correctly_list = answered_correctly_list
return df, sample_df
def __next__(self):
added_user = set()
pre_start = self.current
pre_added_user = -1
pre_task_container_id = -1
pre_content_type_id = -1
user_answer_list = []
answered_correctly_list = []
while self.current < self.len:
crr_user_id = self.user_id[self.current]
crr_task_container_id = self.task_container_id[self.current]
crr_content_type_id = self.content_type_id[self.current]
if crr_user_id in added_user and(crr_user_id != pre_added_user or(
 crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)) :
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 if len(added_user)== self.max_user:
if crr_user_id == pre_added_user and(
 crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
continue
else:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 added_user.add(crr_user_id)
 pre_added_user = crr_user_id
pre_task_container_id = crr_task_container_id
pre_content_type_id = crr_content_type_id
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
if pre_start < self.current:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 else:
raise StopIteration()
if OFFLINE:
target_df = pd.read_pickle(f'{MY_DATA_PATH}/valid.pickle')
 iter_test = Iter_Valid(target_df, max_user=1000)
 predicted = []
def set_predict(df):
predicted.append(df)
 else:
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
previous_test_df = None
idx = 0
for(test_df, sample_prediction_df)in iter_test:
print('idx: ', idx)
 test_df[TARGET] = 0
test_df['prior_question_elapsed_time'] //= 1000
test_df['timestamp'] /= 1000
test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time'] = test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean ).astype(np.int32)
 test_df = reduce_mem_usage(test_df, verbose=True)
 ques_test = test_df.loc[test_df.content_type_id == False, ['row_id', 'content_id']].reset_index(drop=True)
 ques_test = reduce_mem_usage(ques_test, verbose=True)
 if previous_test_df is not None:
', np.array(eval(test_df["prior_group_answers_correct"].iloc[0])))
 previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_user_feats(
 df=previous_test_df, user_content_pos_cnt_dict=user_content_pos_cnt_dict,
user_pos_cnt_dict=user_pos_cnt_dict,
user_part_pos_cnt_dict=user_part_pos_cnt_dict,
user_consecutive_pos_cnt_dict=user_consecutive_pos_cnt_dict,
user_target_win25_dict=user_target_win25_dict,
user_explanation_pos_cnt_dict=user_explanation_pos_cnt_dict,
user_tags_pos_cnt_dict=user_tags_pos_cnt_dict, 
)
 previous_test_df = test_df.copy(deep=True)
 state_feat_test = get_stat_feat(df=test_df.copy(deep=True), feat_cols=static_feat_cols)
 state_feat_test = reduce_mem_usage(state_feat_test, verbose=True)
 user_feat_test = make_user_loop_features(
 df=test_df, content_target_mean_dict=content_target_mean_dict,
user_cnt_dict=user_cnt_dict,
user_pos_cnt_dict=user_pos_cnt_dict,
user_part_cnt_dict=user_part_cnt_dict,
user_part_pos_cnt_dict=user_part_pos_cnt_dict,
user_content_cnt_dict=user_content_cnt_dict,
user_content_pos_cnt_dict=user_content_pos_cnt_dict,
user_content_redo_cnt_dict=user_content_redo_cnt_dict,
user_content_mean_sum_dict=user_content_mean_sum_dict,
user_consecutive_pos_cnt_dict=user_consecutive_pos_cnt_dict,
user_target_win25_dict=user_target_win25_dict,
user_content_mean_win10_dict=user_content_mean_win10_dict,
user_explanation_cnt_dict=user_explanation_cnt_dict,
user_explanation_pos_cnt_dict=user_explanation_pos_cnt_dict,
user_elapse_time_sum_dict=user_elapse_time_sum_dict,
user_elapse_time_win10_dict=user_elapse_time_win10_dict,
user_last_timestamp_dict=user_last_timestamp_dict,
user_last_task_dict=user_last_task_dict,
user_content_win5_dict=user_content_win5_dict,
user_part_win10_dict=user_part_win10_dict,
bundle_state_dict=bundle_state_dict,
user_cum_time_dict=user_cum_time_dict,
user_timespan_win10_dict=user_timespan_win10_dict,
user_tags_cnt_dict=user_tags_cnt_dict,
user_tags_pos_cnt_dict=user_tags_pos_cnt_dict,
user_continue_quest_cnt_dict=user_continue_quest_cnt_dict,
update=False, isTrain=False 
)
 test_feat = merge_features(df=ques_test,
static_feat=state_feat_test,
content_feat=content_feat_test,
content_feat2=content_feat2_test,
user_feat=user_feat_test)
 ques_test[TARGET] = model.predict(test_feat[feat_cols].values.reshape(len(test_feat), len(feat_cols)))
 set_predict(ques_test[['row_id', TARGET]])
 idx += 1
if OFFLINE:
if idx >= 5:
break 
<set_options>
train_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
test_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time']
%%time
def seed_everything(seed = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 torch.manual_seed(seed)
 torch.set_deterministic(False)
 seed_everything()
dtype = {'timestamp': 'int64', 'user_id': 'int32' ,'content_id': 'int16','content_type_id': 'int8','answered_correctly':'int8'}
train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 5, 7], dtype=dtype)
 train_df = train_df[train_df.content_type_id == False]
del train_df['content_type_id']
gc.collect()
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True)
 questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')
 dict_tags = {}
for i, tags in enumerate(questions['tags'].unique()):
if tags in dict_tags.keys() :
continue
dict_tags[tags] = i
questions.tags.replace(dict_tags, inplace=True)
 questions.part = questions.part.astype(np.int8)
 questions.tags = questions.tags.astype(np.int16)
 skills = questions["question_id"].unique()
n_skill = len(skills)
 print("number skills", len(skills))
questions.rename(columns = {'question_id': 'content_id'}, inplace=True)
 train_df = train_df.merge(questions[['content_id', 'part', 'tags']], on='content_id', how='left')
 skills = questions["content_id"].unique()
n_skill = len(skills)
 print("number skills", len(skills))<groupby>
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
print('Loading the training data...')
 train = pd.read_csv(train_path, skiprows=range(1,84903891), dtype=dtypes, nrows=11000000,usecols=train_cols )
%%time
group = train_df.groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values,
r['task_container_id'].values,
r['timestamp'].values,
r['part'].values,
r['tags'].values 
))
del train_df
gc.collect()<define_variables>
len_train = len(train)
 print('The initial size of the train set is', len_train )
MAX_SEQ = 180
ACCEPTED_USER_CONTENT_SIZE = 4
EMBED_SIZE = 128
BATCH_SIZE = 64
DROPOUT = 0.1<define_search_model>
def df_featured(df):
print("Creating new time features: 'hour' and 'day'...")
 df['hour'] = pd.to_datetime(df.click_time ).dt.hour.astype('uint8')
 df['day'] = pd.to_datetime(df.click_time ).dt.day.astype('uint8')
 gc.collect()
print("Feature Engineering
")
 print('1.Computing the number of clicks associated with a given IP address within each hour...')
 n_channel = df[['ip','day','hour','channel']].groupby(by=['ip','day',
'hour'])[['channel']].count().reset_index().rename(columns={'channel': 'n_channels'})
 print('Merging the channels data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','day','hour'], how='left')
 del n_channel
gc.collect()
print('2.Computing the number of clicks associated with a given IP address and app...')
 n_channel = df[['ip','app', 'channel']].groupby(by=['ip',
'app'])[['channel']].count().reset_index().rename(columns={'channel': 'ip_app_count'})
 print('Merging the channels data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','app'], how='left')
 del n_channel
gc.collect()
print('3.Computing the number of clicks associated with a given IP address, app, and os...')
 n_channel = df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app',
'os'])[['channel']].count().reset_index().rename(columns={'channel': 'ip_app_os_count'})
 print('Merging the channels data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','app', 'os'], how='left')
 del n_channel
gc.collect()
print('4.grouping by : ip_day_chl_var_hour.. ')
 n_channel = df[['ip','day','hour','channel']].groupby(by=['ip','day',
'channel'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_tchan_count'})
 print('Merging the hour data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','day','channel'], how='left')
 del n_channel
gc.collect()
print('5.grouping by : ip_app_os_var_hour.. ')
 n_channel = df[['ip','app', 'os', 'hour']].groupby(by=['ip', 'app',
'os'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_app_os_var'})
 print('Merging the hour data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','app', 'os'], how='left')
 del n_channel
gc.collect()
print('6.grouping by : ip_app_channel_var_day...')
 n_channel = df[['ip','app', 'channel', 'day']].groupby(by=['ip', 'app',
'channel'])[['day']].var().reset_index().rename(index=str, columns={'day': 'ip_app_channel_var_day'})
 print('Merging the day data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','app', 'channel'], how='left')
 del n_channel
gc.collect()
print('7.grouping by : ip_app_chl_mean_hour.. ')
 n_channel = df[['ip','app', 'channel','hour']].groupby(by=['ip', 'app',
'channel'])[['hour']].mean().reset_index().rename(index=str, columns={'hour': 'ip_app_channel_mean_hour'})
 print('Merging the meanhour data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','app', 'channel'], how='left')
 del n_channel
gc.collect()
print('8.group by : ip_hh_dev')
 n_channel = df[['ip', 'device', 'hour', 'day', 'channel']].groupby(by=['ip', 'device', 'day',
'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_hh_dev'})
 print('Merging the channel data with the main data set...
 ')
 df = df.merge(n_channel, on=['ip','device','day','hour'], how='left')
 del n_channel
gc.collect()
df['n_channels'] = df['n_channels'].astype('uint16')
 df['ip_app_count'] = df['ip_app_count'].astype('uint16')
 df['ip_app_os_count'] = df['ip_app_os_count'].astype('uint16')
 df['ip_tchan_count'] = df['ip_tchan_count'].astype('float32')
 df['ip_app_os_var'] = df['ip_app_os_var'].astype('float32')
 df['ip_app_channel_var_day'] = df['ip_app_channel_var_day'].astype('float32')
 df['ip_app_channel_mean_hour'] = df['ip_app_channel_mean_hour'].astype('float32')
 df['nip_hh_dev'] = df['nip_hh_dev'].astype('uint16')
 df.drop(['ip','day'], axis=1, inplace=True)
 gc.collect()
return df
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(shape):
future_mask = np.triu(np.ones(shape), k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SubLayer(nn.Module):
def __init__(self,embed_dim):
super().__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 def forward(self, e, x):
att_mask = future_mask(shape=(e.size(0), x.size(0)) ).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 return x, att_weight
class BERTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super().__init__()
self.pos_embedding = nn.Embedding(max_seq, embed_dim)
 self.embedding = nn.Embedding(n_skill+1, embed_dim)
 self.ans_embedding = nn.Embedding(3, embed_dim)
 self.time_embedding = nn.Embedding(10000, embed_dim)
 self.lag_time_embedding = nn.Embedding(3600, embed_dim)
 self.elapsed_time_embedding = nn.Embedding(1520, embed_dim)
 self.part_embedding = nn.Embedding(8, embed_dim)
 self.sub1 = SubLayer(embed_dim)
 self.fc = nn.Linear(embed_dim, embed_dim*2)
 self.fc1 = nn.Linear(embed_dim*5, embed_dim)
 self.bacth_norm = nn.BatchNorm1d(max_seq)
 self.bacth_norm1 = nn.BatchNorm1d(max_seq)
 self.pred = nn.Linear(embed_dim*2, 1)
 def forward(self, history_question, history_answer, time, lag_time, part, elapsed_time):
device = history_question.device
history_answer = history_answer
history_answer = self.ans_embedding(history_answer)
 x = self.embedding(history_question)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 time_x = self.time_embedding(time)
 lag_time = self.lag_time_embedding(lag_time)
 part = self.part_embedding(part)
 elapsed_time = self.elapsed_time_embedding(elapsed_time)
 history_answer += pos_x
x += history_answer
time_x += history_answer
lag_time += history_answer
part += history_answer
elapsed_time += history_answer
x = torch.cat([x, time_x, lag_time, part, elapsed_time], axis=-1)
 x = self.fc1(x)
 x = x.permute(1, 0, 2)
 x, att_weight= self.sub1(x, x)
 x = self.fc(x)
 x = self.bacth_norm(x)
 x = self.pred(x)
 return x.squeeze(-1), att_weight<load_pretrained>
metrics = 'auc'
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric':metrics,
'learning_rate': 0.05,
'num_leaves': 31,
'max_depth': -1,
'min_child_samples': 20,
'max_bin': 255,
'subsample': 0.7,
'subsample_freq': 1,
'colsample_bytree': 0.7,
'subsample_for_bin': 200000,
'min_child_weight': 5,
'min_split_gain': 0,
'nthread': 8,
'verbose': 0,
'scale_pos_weight':99.7,
'metric':metrics
}
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = torch.load('/kaggle/input/v9-test/7901_bert_v4.pth', map_location=lambda storage, loc: storage.cuda(0))<split>
target = 'is_attributed'
train[target] = train[target].astype('uint8')
 predictors = ['app','device', 'os', 'channel', 'hour', 'n_channels', 'ip_app_count', 'ip_app_os_count',
'ip_tchan_count','ip_app_os_var','ip_app_channel_var_day','ip_app_channel_mean_hour','nip_hh_dev']
categorical = [ 'app', 'device', 'os', 'channel']
gc.collect()
<feature_engineering><EOS> 
VALIDATE = False
MAX_ROUNDS = 1000
EARLY_STOP = 50
OPT_ROUNDS = 1000
FULL_OUTFILE = 'sub_lgbm1.csv'
VALID_OUTFILE = 'sub_lgbm_v.csv'
if VALIDATE:
train, val_df = train_test_split(train, train_size=.95, shuffle=False)
 print("
Train Information ", train.info())
 print("
Val Information ", val_df.info())
 print("train size: ", len(train))
print("valid size: ", len(val_df))
gc.collect()
print("Training...
 ")
 num_boost_round=MAX_ROUNDS
early_stopping_rounds=EARLY_STOP
xgtrain = lgb.Dataset(train[predictors].values, label=train[target].values,
feature_name=predictors,
categorical_feature=categorical 
)
 del train
gc.collect()
xgvalid = lgb.Dataset(val_df[predictors].values, label=val_df[target].values,
feature_name=predictors,
categorical_feature=categorical 
)
 del val_df
gc.collect()
evals_results = {}
bst = lgb.train(lgb_params,
xgtrain,
valid_sets= [xgvalid],
valid_names=['valid'],
evals_result=evals_results,
num_boost_round=num_boost_round,
early_stopping_rounds=early_stopping_rounds,
verbose_eval=10,
feval=None)
 n_estimators = bst.best_iteration
print("
Model Report")
 print("n_estimators : ", n_estimators)
 print(metrics+":", evals_results['valid'][metrics][n_estimators-1])
 outfile = VALID_OUTFILE
del xgvalid
else:
print("train size: ", len(train))
gc.collect()
print("Training...")
 num_boost_round=OPT_ROUNDS
xgtrain = lgb.Dataset(train[predictors].values, label=train[target].values,
feature_name=predictors,
categorical_feature=categorical 
)
 del train
gc.collect()
print("Dataset preparing done")
 bst = lgb.train(lgb_params,
xgtrain,
num_boost_round=num_boost_round,
verbose_eval=10,
feval=None)
 print("Traing done")
 outfile = FULL_OUTFILE
del xgtrain
gc.collect()
print('load test...')
 test_cols = ['ip','app','device','os', 'channel', 'click_time', 'click_id']
test_df = pd.read_csv(test_path, dtype=dtypes, usecols=test_cols)
 test_df = df_featured(test_df)
 gc.collect()
sub = pd.DataFrame()
sub['click_id'] = test_df['click_id']
print("Predicting...")
 sub['is_attributed'] = bst.predict(test_df[predictors])
 print("writing...")
 sub.to_csv(outfile, index=False, float_format='%.9f')
 print("done...")
 print(sub.info() )