Kaggle competition: Mechanisms of Action (MoA) Prediction
pd.options.display.max_columns = 99
prev_test_df = None
for idx,(test_df, _)in tqdm(enumerate(iter_test)) :
if prev_test_df is not None:
prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 update_user_data(state, prev_test_df)
 if idx < 4:
display(test_df)
 display(prev_test_df)
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value = False ).astype(bool)
 test_df['prior_question_elapsed_time'].fillna(23916, inplace=True)
 test_df = features_questions_df.merge(test_df, how='right', on='content_id')
 add_new_users(test_df)
 add_attempt_retry(test_df)
 add_mean_content_accuracy(test_df)
 user_data = get_user_data(state, test_df)
 for feature, values in user_data.items() :
test_df[feature] = values
test_df['hmean_user_content_accuracy'] = 2 *(
(test_df['mean_user_accuracy'] * test_df['mean_content_accuracy'])/ 
(test_df['mean_user_accuracy'] + test_df['mean_content_accuracy'])
)
 test_df['content_id_pos_diff'] =(test_df['answered_user'] - test_df['mean_content_id_pos'] ).values.astype(np.int16)
 test_df['answered_correctly'] = model.predict(test_df[features])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])
 prev_test_df = test_df.copy()<load_from_csv>
warnings.filterwarnings("ignore")
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )
submission = pd.read_csv('./submission.csv' )<set_options>
def get_logger(filename='log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=f"{filename}.log")
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
logger = get_logger()
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100 )<load_pretrained>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
 def pickle_dump_dill(obj, path):
with open(path, mode='wb')as f:
dill.dump(obj,f)
 def pickle_load_dill(path):
with open(path, mode='rb')as f:
data = dill.load(f)
 return data
def pickle_dump(obj, path):
with open(path, mode='wb')as f:
pickle.dump(obj,f)
 def pickle_load(path):
with open(path, mode='rb')as f:
data = pickle.load(f)
 return data
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<load_pretrained>
train = train_features.merge(train_targets_scored, on='sig_id')
 target_cols = [c for c in train_targets_scored.columns if c not in ['sig_id']]
cols = target_cols + ['cp_type']
train[cols].groupby('cp_type' ).sum().sum(1 )
train_pickle = '.. /input/riiid-cross-validation-files/cv1_train.pickle'
valid_pickle = '.. /input/riiid-cross-validation-files/cv1_valid.pickle'
feld_needed = ['timestamp','user_id', 'content_id', 'content_type_id', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation']
train = pd.read_pickle(train_pickle)[feld_needed]
valid = pd.read_pickle(valid_pickle)[feld_needed]<split>
folds = train.copy()
Fold = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[target_cols])) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 print(folds.shape )
kaggle=True
if kaggle:
train=train[:10000]
valid=valid[:10000]<data_type_conversions>
class TrainDataset(Dataset):
def __init__(self, df, num_features, cat_features, labels):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
self.labels = labels
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 label = torch.tensor(self.labels[idx] ).float()
return cont_x, cate_x, label
class TestDataset(Dataset):
def __init__(self, df, num_features, cat_features):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 return cont_x, cate_x
train.timestamp=(train.timestamp/1000 ).astype('int32')
 valid.timestamp=(valid.timestamp/1000 ).astype('int32')
 train = reduce_mem_usage(train, use_float16=False)
 valid = reduce_mem_usage(valid, use_float16=False )<categorify>
cat_features = ['cp_time', 'cp_dose']
num_features = [c for c in train.columns if train.dtypes[c] != 'object']
num_features = [c for c in num_features if c not in cat_features]
num_features = [c for c in num_features if c not in target_cols]
target = train[target_cols].values
def cate2num(df):
df['cp_time'] = df['cp_time'].map({24: 0, 48: 1, 72: 2})
 df['cp_dose'] = df['cp_dose'].map({'D1': 3, 'D2': 4})
 return df
train = cate2num(train)
 test = cate2num(test )
def add_prior_b(df, questions_cb, questions_bpt, prior_b_dict, verbose=True):
df = df.merge(questions_cb, how = 'left', on = 'content_id')
 df['bundle_id'] =(df['bundle_id'].fillna(99999)).astype('int32')
 prior_b = np.zeros(len(df), dtype=np.int32)
 for cnt,row in(enumerate(tqdm(df[['user_id','content_type_id','bundle_id']].values)) if verbose else enumerate(df[['user_id','content_type_id','bundle_id']].values)) :
user_id=int(row[0])
 content_type_id=int(row[1])
 bundle_id=int(row[2])
 if prior_b_dict[user_id][0] != bundle_id:
prior_b[cnt] = prior_b_dict[user_id][0]
if(content_type_id == 0):
prior_b_dict[user_id][1] = prior_b_dict[user_id][0]
prior_b_dict[user_id][0] = bundle_id
else:
prior_b[cnt] = prior_b_dict[user_id][1]
df['prior_bundle']=prior_b
df = df.merge(questions_bpt, how = 'left', on = 'prior_bundle')
 df['prior_bundle'] = df['prior_bundle'].fillna(99999)
 df['prior_part'] =(df['prior_part'].fillna(9)).astype('int8')
 df['prior_taggroup'] =(df['prior_taggroup'].fillna(99)).astype('int16')
 return df<load_from_csv>
class CFG:
max_grad_norm=1000
gradient_accumulation_steps=1
hidden_size=512
dropout=0.4
lr=1e-3
weight_decay=1e-5
batch_size=128
epochs=50
num_features=num_features
cat_features=cat_features
target_cols=target_cols
questions = pd.read_csv('.. /input/riid-preprocessed/questions_with_taggroup.csv')
 questions_cb=questions[['question_id', 'bundle_id']].copy()
questions_cb = reduce_mem_usage(questions_cb, use_float16=False)
 questions_cb.columns=['content_id', 'bundle_id']
questions_bpt=questions[['bundle_id', 'part', 'taggroup']].copy()
questions_bpt = reduce_mem_usage(questions_bpt, use_float16=False)
 questions_bpt.columns=['prior_bundle', 'prior_part', 'prior_taggroup']
questions_bpt.drop_duplicates(subset='prior_bundle', keep='first', inplace=True)
 prior_b_dict = defaultdict(lambda:defaultdict(lambda:99999))
train = add_prior_b(train, questions_cb, questions_bpt, prior_b_dict)
 valid = add_prior_b(valid, questions_cb, questions_bpt, prior_b_dict )<groupby>
class TabularNN(nn.Module):
def __init__(self, cfg):
super().__init__()
self.mlp = nn.Sequential(
 nn.Linear(len(cfg.num_features), cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU(cfg.hidden_size),
nn.Linear(cfg.hidden_size, cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU(cfg.hidden_size),
nn.Linear(cfg.hidden_size, len(cfg.target_cols)) , 
)
 def forward(self, cont_x, cate_x):
x = self.mlp(cont_x)
 return x
create_content_df=False
if create_content_df:
content_df = train[train.content_type_id==0].groupby('content_id' ).\
agg({'answered_correctly': ['mean','sum', 'count', 'std']} ).reset_index()
content_df.columns = ['content_id','acc_avg_c','acc_sum_c', 'count_c', 'acc_std_c']
bundle_df = train[train.content_type_id==0].groupby('prior_bundle' ).\
agg({'prior_question_had_explanation': ['mean','sum', 'count', 'std'],
'prior_question_elapsed_time': ['mean','sum', 'count', 'std']} ).reset_index()
bundle_df.columns = ['bundle_id','exp_avg_b', 'exp_sum_b', 'exp_count_b','exp_std_b','time_avg_b', 'time_sum_b', 'time_count_b','time_std_b']
questions.columns=['content_id', 'bundle_id', 'correct_answer', 'part', 'tags', 'taggroup']
content_df=content_df.merge(questions, how = 'left', on = 'content_id')
 content_df=content_df.merge(bundle_df, how = 'left', on = 'bundle_id')
 content_df.loc[10033,'tags']='188'
inv_rows = content_df[content_df['tags'].apply(type)== float].index
content_df.at[inv_rows, 'tags'] = ''
content_df['tags'] = content_df['tags'].apply(lambda x: np.array(x.split() ).astype(int))
tag_df=pd.DataFrame()
tag_df['tags']=range(189)
 tag_columns=['acc_sum_c', 'count_c', 'exp_sum_b', 'exp_count_b','time_sum_b', 'time_count_b']
tag_df[tag_columns]=0
for i in range(len(content_df)) :
tags=content_df.loc[i,'tags']
tag_df.loc[tags,tag_columns]=tag_df.loc[tags,tag_columns]+content_df.loc[i,tag_columns]
tag_df=tag_df.astype(int)
 tag_df['acc_avg_t']=tag_df['acc_sum_c']/tag_df['count_c']
tag_df['exp_avg_t']=tag_df['exp_sum_b']/tag_df['exp_count_b']
tag_df['time_avg_t']=tag_df['time_sum_b']/tag_df['time_count_b']
for i in range(len(content_df)) :
tags=content_df.loc[i,'tags']
content_df.loc[i,'acc_avg_t']=tag_df.loc[tags].mean() ['acc_avg_t']
content_df.loc[i,'exp_avg_t']=tag_df.loc[tags].mean() ['exp_avg_t']
content_df.loc[i,'time_avg_t']=tag_df.loc[tags].mean() ['time_avg_t']
content_df['type_of']=4
content_df['col_for_merge']=(content_df['content_id'].astype('int32')*10 ).astype('int32')
 lectures = pd.read_csv('/media/hama/ssd2/inubushi/kaggle/riid/riiid-test-answer-prediction/lectures.csv')
 le = LabelEncoder()
le.fit(lectures['type_of'])
 lectures['type_of'] = le.transform(lectures['type_of'])
 lectures['col_for_merge']=(lectures['lecture_id']*10+1 ).astype('int32')
 content_df=lectures.append(content_df)
 train = train.merge(questions[['content_id', 'part']], how = 'left', on = 'content_id')
 part_df = train[train.content_type_id==0].groupby('part' ).\
agg({'answered_correctly': ['mean','sum', 'count', 'std']} ).reset_index()
part_df.columns = ['part','acc_avg_p','acc_sum_p','count_p','acc_std_p']
part_df2 = train[train.content_type_id==0].groupby('prior_part' ).\
agg({'prior_question_had_explanation': ['mean','sum', 'count', 'std'],
'prior_question_elapsed_time': ['mean','sum', 'count', 'std']} ).reset_index()
part_df2.columns = ['part', 'exp_avg_p','exp_sum_p','exp_count_p','exp_std_p', 'time_avg_p','time_sum_p','time_count_p','time_std_p']
part_df=part_df.merge(part_df2, how = 'left', on = 'part')
 train=train.drop('part', axis=1)
 content_df = content_df.merge(part_df, how = 'left', on = 'part')
 train = train.merge(questions[['content_id', 'taggroup']], how = 'left', on = 'content_id')
 taggroup_df = train[train.content_type_id==0].groupby('taggroup' ).\
agg({'answered_correctly': ['mean','sum', 'count', 'std']} ).reset_index()
taggroup_df.columns = ['taggroup','acc_avg_tg','acc_sum_tg','count_tg','acc_std_tg']
taggroup_df2 = train[train.content_type_id==0].groupby('prior_taggroup' ).\
agg({'prior_question_had_explanation': ['mean','sum', 'count', 'std'],
'prior_question_elapsed_time': ['mean','sum', 'count', 'std']} ).reset_index()
taggroup_df2.columns = ['taggroup', 'exp_avg_tg','exp_sum_tg','exp_count_tg','exp_std_tg', 'time_avg_tg','time_sum_tg','time_count_tg','time_std_tg']
taggroup_df=taggroup_df.merge(taggroup_df2, how = 'left', on = 'taggroup')
 train=train.drop('taggroup', axis=1)
 content_df = content_df.merge(taggroup_df, how = 'left', on = 'taggroup')
 content_df = content_df.fillna(0)
 content_df.to_csv('content_df.csv', index=False)
 else:
content_df = pd.read_csv('.. /input/riid-preprocessed/content_df_all.csv' )<drop_column>
def train_fn(train_loader, model, optimizer, epoch, scheduler, device):
losses = AverageMeter()
model.train()
for step,(cont_x, cate_x, y)in enumerate(train_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters() , CFG.max_grad_norm)
 if(step + 1)% CFG.gradient_accumulation_steps == 0:
scheduler.step()
optimizer.step()
optimizer.zero_grad()
return losses.avg
def validate_fn(valid_loader, model, device):
losses = AverageMeter()
model.eval()
val_preds = []
for step,(cont_x, cate_x, y)in enumerate(valid_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 val_preds.append(pred.sigmoid().detach().cpu().numpy())
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
val_preds = np.concatenate(val_preds)
 return losses.avg, val_preds
def inference_fn(test_loader, model, device):
model.eval()
preds = []
for step,(cont_x, cate_x)in enumerate(test_loader):
cont_x, cate_x = cont_x.to(device), cate_x.to(device)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
content_df=content_df[['col_for_merge', 'part', 'taggroup', 'acc_avg_t', 'exp_avg_t', 'time_avg_t',
'acc_avg_p','acc_sum_p','count_p','acc_std_p',
'exp_avg_p','exp_sum_p','exp_std_p',
'time_avg_p','time_sum_p','time_std_p',
'acc_avg_c','acc_sum_c', 'count_c','acc_std_c',
'exp_avg_b', 'exp_sum_b', 'exp_std_b',
'time_avg_b','time_sum_b','time_std_b',
'acc_avg_tg','acc_sum_tg','count_tg','acc_std_tg',
'exp_avg_tg','exp_sum_tg','exp_std_tg',
'time_avg_tg','time_sum_tg','time_std_tg']]
train['col_for_merge']=(train['content_id'].astype('int32')*10+train['content_type_id'] ).astype('int32')
 valid['col_for_merge']=(valid['content_id'].astype('int32')*10+valid['content_type_id'] ).astype('int32')
 train = train.merge(content_df, how = 'left', on = 'col_for_merge')
 valid = valid.merge(content_df, how = 'left', on = 'col_for_merge' )<data_type_conversions>
def run_single_nn(cfg, train, test, folds, num_features, cat_features, target, device, fold_num=0, seed=42):
logger.info(f'Set seed {seed}')
 seed_everything(seed=seed)
 trn_idx = folds[folds['fold'] != fold_num].index
val_idx = folds[folds['fold'] == fold_num].index
train_folds = train.loc[trn_idx].reset_index(drop=True)
 valid_folds = train.loc[val_idx].reset_index(drop=True)
 train_target = target[trn_idx]
valid_target = target[val_idx]
train_dataset = TrainDataset(train_folds, num_features, cat_features, train_target)
 valid_dataset = TrainDataset(valid_folds, num_features, cat_features, valid_target)
 train_loader = DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True,
num_workers=4, pin_memory=True, drop_last=True)
 valid_loader = DataLoader(valid_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True, drop_last=False)
 model = TabularNN(cfg)
 model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=cfg.lr, weight_decay=cfg.weight_decay)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.2, div_factor=1e4,
max_lr=1e-2, epochs=cfg.epochs, steps_per_epoch=len(train_loader))
log_df = pd.DataFrame(columns=(['EPOCH']+['TRAIN_LOSS']+['VALID_LOSS']))
best_loss = np.inf
for epoch in range(cfg.epochs):
train_loss = train_fn(train_loader, model, optimizer, epoch, scheduler, device)
 valid_loss, val_preds = validate_fn(valid_loader, model, device)
 log_row = {'EPOCH': epoch,
'TRAIN_LOSS': train_loss,
'VALID_LOSS': valid_loss,
}
log_df = log_df.append(pd.DataFrame(log_row, index=[0]), sort=False)
 if valid_loss < best_loss:
logger.info(f'epoch{epoch} save best model...{valid_loss}')
 best_loss = valid_loss
oof = np.zeros(( len(train), len(cfg.target_cols)))
 oof[val_idx] = val_preds
torch.save(model.state_dict() , f"fold{fold_num}_seed{seed}.pth")
 test_dataset = TestDataset(test, num_features, cat_features)
 test_loader = DataLoader(test_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True)
 model = TabularNN(cfg)
 model.load_state_dict(torch.load(f"fold{fold_num}_seed{seed}.pth"))
model.to(device)
 predictions = inference_fn(test_loader, model, device)
 torch.cuda.empty_cache()
return oof, predictions
def run_kfold_nn(cfg, train, test, folds, num_features, cat_features, target, device, n_fold=5, seed=42):
oof = np.zeros(( len(train), len(cfg.target_cols)))
 predictions = np.zeros(( len(test), len(cfg.target_cols)))
 for _fold in range(n_fold):
logger.info("Fold {}".format(_fold))
_oof, _predictions = run_single_nn(cfg,
train,
test,
folds,
num_features,
cat_features,
target,
device,
fold_num=_fold,
seed=seed)
 oof += _oof
predictions += _predictions / n_fold
score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"CV score: {score}")
 return oof, predictions
train['prior_question_elapsed_time'] = train['prior_question_elapsed_time'].fillna(25439.41)
 train['prior_question_had_explanation'] = train['prior_question_had_explanation'].fillna(False ).astype('int8')
 valid['prior_question_elapsed_time'] = valid['prior_question_elapsed_time'].fillna(25439.41)
 valid['prior_question_had_explanation'] = valid['prior_question_had_explanation'].fillna(False ).astype('int8')
 train = reduce_mem_usage(train, use_float16=False)
 valid = reduce_mem_usage(valid, use_float16=False )<compute_test_metric>
oof = np.zeros(( len(train), len(CFG.target_cols)))
 predictions = np.zeros(( len(test), len(CFG.target_cols)))
 SEED = [0, 1, 2]
for seed in SEED:
_oof, _predictions = run_kfold_nn(CFG,
train, test, folds,
num_features, cat_features, target,
device,
n_fold=5, seed=seed)
 oof += _oof / len(SEED)
 predictions += _predictions / len(SEED)
 score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"Seed Averaged CV score: {score}" )
GAMMA=0.75
GAMMA2=0.25
def update_std(new_value, old_std, old_sum, old_count):
if old_count > 0:
old_avg = old_sum/old_count
else:
old_avg = 0
new_avg =(old_sum+new_value)/(old_count+1)
 new_std =(( old_count*(old_std**2+old_avg**2)+new_value**2)/(old_count+1)-new_avg**2)**(1/2)
 return new_std
def add_feats(df, acc_sum_u_dict, acc_std_u_dict, acc_hist_u_dict, acc_hist2_u_dict,
exp_sum_u_dict, exp_std_u_dict, exp_hist_u_dict, exp_hist2_u_dict,
time_sum_u_dict, time_std_u_dict, time_hist_u_dict, time_hist2_u_dict,
dpt_sum_u_dict, dpt_std_u_dict, dpt_hist_u_dict, dpt_hist2_u_dict,
dif_sum_u_dict, dif_std_u_dict, dif_hist_u_dict, dif_hist2_u_dict,
count_u_dict, count_lecture_u_dict,
last_time_u_dict, last_time2_u_dict, last_time3_u_dict,
last_time4_u_dict, last_time5_u_dict, last_time6_u_dict,
tfl_std_u_dict, tfl_hist_u_dict, tfl_hist2_u_dict,
acc_sum_c_dict, exp_sum_b_dict, time_sum_b_dict,
acc_std_c_dict, exp_std_b_dict, time_std_b_dict,
count_c_dict, count_b_dict,
acc_sum_up_dict, acc_hist_up_dict, acc_hist2_up_dict,
count_up_dict, count_lecture_up_dict, last_time_up_dict, last_time2_up_dict,
tfl_hist_up_dict,tfl_hist2_up_dict,
exp_sum_up_dict, exp_hist_up_dict, exp_hist2_up_dict,
time_sum_up_dict, time_hist_up_dict, time_hist2_up_dict,
count_upp_dict,
acc_sum_uc_dict, acc_hist_uc_dict, acc_hist2_uc_dict,
count_uc_dict, count_lecture_uc_dict, last_time_uc_dict, last_time2_uc_dict,
tfl_hist_uc_dict, tfl_hist2_uc_dict,
exp_sum_ub_dict, exp_hist_ub_dict, exp_hist2_ub_dict,
time_sum_ub_dict, time_hist_ub_dict, time_hist2_ub_dict,
count_ub_dict):
acsu = np.zeros(len(df), dtype=np.int16)
 acstdu = np.zeros(len(df), dtype=np.float32)
 achu = np.zeros(len(df), dtype=np.float32)
 achu2 = np.zeros(len(df), dtype=np.float32)
 exsu = np.zeros(len(df), dtype=np.int16)
 exstdu = np.zeros(len(df), dtype=np.float32)
 exhu = np.zeros(len(df), dtype=np.float32)
 exhu2 = np.zeros(len(df), dtype=np.float32)
 tisu = np.zeros(len(df), dtype=np.int32)
 tistdu = np.zeros(len(df), dtype=np.float32)
 tihu = np.zeros(len(df), dtype=np.float32)
 tihu2 = np.zeros(len(df), dtype=np.float32)
 dpsu = np.zeros(len(df), dtype=np.float32)
 dpstdu = np.zeros(len(df), dtype=np.float32)
 dphu = np.zeros(len(df), dtype=np.float32)
 dphu2 = np.zeros(len(df), dtype=np.float32)
 disu = np.zeros(len(df), dtype=np.float32)
 distdu = np.zeros(len(df), dtype=np.float32)
 dihu = np.zeros(len(df), dtype=np.float32)
 dihu2 = np.zeros(len(df), dtype=np.float32)
 cu = np.zeros(len(df), dtype=np.int16)
 clu = np.zeros(len(df), dtype=np.int16)
 tflu = np.zeros(len(df), dtype=np.int32)
 tflu2 = np.zeros(len(df), dtype=np.int32)
 tflu3 = np.zeros(len(df), dtype=np.int32)
 tflu4 = np.zeros(len(df), dtype=np.int32)
 tflu5 = np.zeros(len(df), dtype=np.int32)
 tflstdu = np.zeros(len(df), dtype=np.float32)
 tflhu = np.zeros(len(df), dtype=np.int32)
 tflhu2 = np.zeros(len(df), dtype=np.int32)
 acsc = np.zeros(len(df), dtype=np.int32)
 acstdc = np.zeros(len(df), dtype=np.float32)
 exsb = np.zeros(len(df), dtype=np.int32)
 exstdb = np.zeros(len(df), dtype=np.float32)
 tisb = np.zeros(len(df), dtype=np.int32)
 tistdb = np.zeros(len(df), dtype=np.float32)
 cc = np.zeros(len(df), dtype=np.int32)
 cb = np.zeros(len(df), dtype=np.int32)
 acsup = np.zeros(len(df), dtype=np.int16)
 achup = np.zeros(len(df), dtype=np.float32)
 achup2 = np.zeros(len(df), dtype=np.float32)
 cup = np.zeros(len(df), dtype=np.int16)
 clup = np.zeros(len(df), dtype=np.int16)
 tflup = np.zeros(len(df), dtype=np.int32)
 tflhup = np.zeros(len(df), dtype=np.int32)
 tflhup2 = np.zeros(len(df), dtype=np.int32)
 exsup = np.zeros(len(df), dtype=np.int16)
 exhup = np.zeros(len(df), dtype=np.float32)
 exhup2 = np.zeros(len(df), dtype=np.float32)
 tisup = np.zeros(len(df), dtype=np.int32)
 tihup = np.zeros(len(df), dtype=np.float32)
 tihup2 = np.zeros(len(df), dtype=np.float32)
 cupp = np.zeros(len(df), dtype=np.int32)
 cup1 = np.zeros(len(df), dtype=np.int16)
 cup2 = np.zeros(len(df), dtype=np.int16)
 cup3 = np.zeros(len(df), dtype=np.int16)
 cup4 = np.zeros(len(df), dtype=np.int16)
 cup5 = np.zeros(len(df), dtype=np.int16)
 cup6 = np.zeros(len(df), dtype=np.int16)
 cup7 = np.zeros(len(df), dtype=np.int16)
 acsuc = np.zeros(len(df), dtype=np.int16)
 achuc = np.zeros(len(df), dtype=np.float32)
 achuc2 = np.zeros(len(df), dtype=np.float32)
 cuc = np.zeros(len(df), dtype=np.int16)
 cluc = np.zeros(len(df), dtype=np.int16)
 tfluc = np.zeros(len(df), dtype=np.int32)
 tflhuc = np.zeros(len(df), dtype=np.int32)
 tflhuc2 = np.zeros(len(df), dtype=np.int32)
 exsub = np.zeros(len(df), dtype=np.int32)
 exhub = np.zeros(len(df), dtype=np.float32)
 exhub2 = np.zeros(len(df), dtype=np.float32)
 tisub = np.zeros(len(df), dtype=np.int32)
 tihub = np.zeros(len(df), dtype=np.float32)
 tihub2 = np.zeros(len(df), dtype=np.float32)
 cub = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(tqdm(df[['content_type_id',
'answered_correctly',
'prior_question_had_explanation',
'prior_question_elapsed_time',
'acc_avg_c',
'user_id',
'part',
'prior_part',
'content_id',
'bundle_id',
'prior_bundle',
'timestamp']].values)) :
content_type_id=int(row[0])
 answered_correctly=int(row[1])
 prior_question_had_explanation=int(row[2])
 prior_question_elapsed_time=int(row[3])
 acc_avg_c=float(row[4])
 user_id=int(row[5])
 part=int(row[6])
 prior_part=int(row[7])
 content_id=int(row[8])
 bundle_id=int(row[9])
 prior_bundle=int(row[10])
 timestamp=int(row[11])
 acsu[cnt] = acc_sum_u_dict[user_id]
acstdu[cnt] = acc_std_u_dict[user_id]
achu[cnt] = acc_hist_u_dict[user_id]
achu2[cnt] = acc_hist2_u_dict[user_id]
exsu[cnt] = exp_sum_u_dict[user_id]
exstdu[cnt] = exp_std_u_dict[user_id]
exhu[cnt] = exp_hist_u_dict[user_id]
exhu2[cnt] = exp_hist2_u_dict[user_id]
tisu[cnt] = time_sum_u_dict[user_id]
tistdu[cnt] = time_std_u_dict[user_id]
tihu[cnt] = time_hist_u_dict[user_id]
tihu2[cnt] = time_hist2_u_dict[user_id]
dpsu[cnt] = dpt_sum_u_dict[user_id]
dpstdu[cnt] = dpt_std_u_dict[user_id]
dphu[cnt] = dpt_hist_u_dict[user_id]
dphu2[cnt] = dpt_hist2_u_dict[user_id]
disu[cnt] = dif_sum_u_dict[user_id]
distdu[cnt] = dif_std_u_dict[user_id]
dihu[cnt] = dif_hist_u_dict[user_id]
dihu2[cnt] = dif_hist2_u_dict[user_id]
cu[cnt] = count_u_dict[user_id]
clu[cnt] = count_lecture_u_dict[user_id]
tflu[cnt] = timestamp-last_time_u_dict[user_id]
tflu2[cnt] = last_time_u_dict[user_id]-last_time2_u_dict[user_id]
tflu3[cnt] = last_time2_u_dict[user_id]-last_time3_u_dict[user_id]
tflu4[cnt] = last_time3_u_dict[user_id]-last_time4_u_dict[user_id]
tflu5[cnt] = last_time4_u_dict[user_id]-last_time5_u_dict[user_id]
tflstdu[cnt] = update_std(timestamp-last_time_u_dict[user_id], tfl_std_u_dict[user_id], last_time_u_dict[user_id], count_u_dict[user_id])
 tflhu[cnt] = timestamp-last_time_u_dict[user_id]+tfl_hist_u_dict[user_id]*GAMMA
tflhu2[cnt] = timestamp-last_time_u_dict[user_id]+tfl_hist2_u_dict[user_id]*GAMMA2
acsc[cnt] = acc_sum_c_dict[content_id]
acstdc[cnt] = acc_std_c_dict[content_id]
exsb[cnt] = exp_sum_b_dict[bundle_id]
exstdb[cnt] = exp_std_b_dict[bundle_id]
tisb[cnt] = time_sum_b_dict[bundle_id]
tistdb[cnt] = time_std_b_dict[bundle_id]
cc[cnt] = count_c_dict[content_id]
cb[cnt] = count_b_dict[bundle_id]
acsup[cnt] = acc_sum_up_dict[user_id][part]
achup[cnt] = acc_hist_up_dict[user_id][part]
achup2[cnt] = acc_hist2_up_dict[user_id][part]
cup[cnt] = count_up_dict[user_id][part]
clup[cnt] = count_lecture_up_dict[user_id][part]
if count_up_dict[user_id][part]>0:
tflup[cnt] = timestamp-last_time_up_dict[user_id][part]
tflhup[cnt] = timestamp-last_time_up_dict[user_id][part]+tfl_hist_up_dict[user_id][part]*GAMMA
tflhup2[cnt] = timestamp-last_time_up_dict[user_id][part]+tfl_hist2_up_dict[user_id][part]*GAMMA2
exsup[cnt] = exp_sum_up_dict[user_id][part]
exhup[cnt] = exp_hist_up_dict[user_id][part]
exhup2[cnt] = exp_hist2_up_dict[user_id][part]
tisup[cnt] = time_sum_up_dict[user_id][part]
tihup[cnt] = time_hist_up_dict[user_id][part]
tihup2[cnt] = time_hist2_up_dict[user_id][part]
cupp[cnt] = count_upp_dict[user_id][part]
cup1[cnt] = count_up_dict[user_id][1]
cup2[cnt] = count_up_dict[user_id][2]
cup3[cnt] = count_up_dict[user_id][3]
cup4[cnt] = count_up_dict[user_id][4]
cup5[cnt] = count_up_dict[user_id][5]
cup6[cnt] = count_up_dict[user_id][6]
cup7[cnt] = count_up_dict[user_id][7]
acsuc[cnt] = acc_sum_uc_dict[user_id][content_id]
achuc[cnt] = acc_hist_uc_dict[user_id][content_id]
achuc2[cnt] = acc_hist2_uc_dict[user_id][content_id]
cuc[cnt] = count_uc_dict[user_id][content_id]
cluc[cnt] = count_lecture_uc_dict[user_id][content_id]
if count_uc_dict[user_id][content_id]>0:
tfluc[cnt] = timestamp-last_time_uc_dict[user_id][content_id]
tflhuc[cnt] = timestamp-last_time_uc_dict[user_id][content_id]+tfl_hist_uc_dict[user_id][content_id]*GAMMA
tflhuc2[cnt] = timestamp-last_time_uc_dict[user_id][content_id]+tfl_hist2_uc_dict[user_id][content_id]*GAMMA2
exsub[cnt] = exp_sum_ub_dict[user_id][bundle_id]
exhub[cnt] = exp_hist_ub_dict[user_id][bundle_id]
exhub2[cnt] = exp_hist2_ub_dict[user_id][bundle_id]
tisub[cnt] = time_sum_ub_dict[user_id][bundle_id]
tihub[cnt] = time_hist_ub_dict[user_id][bundle_id]
tihub2[cnt] = time_hist2_ub_dict[user_id][bundle_id]
cub[cnt] = count_ub_dict[user_id][bundle_id]
if last_time_u_dict[user_id] == timestamp:
tflu[cnt] = timestamp-last_time2_u_dict[user_id]
tflu2[cnt] = last_time2_u_dict[user_id]-last_time3_u_dict[user_id]
tflu3[cnt] = last_time3_u_dict[user_id]-last_time4_u_dict[user_id]
tflu4[cnt] = last_time4_u_dict[user_id]-last_time5_u_dict[user_id]
tflu5[cnt] = last_time5_u_dict[user_id]-last_time6_u_dict[user_id]
tflstdu[cnt] = update_std(timestamp-last_time2_u_dict[user_id], tfl_std_u_dict[user_id], last_time2_u_dict[user_id], count_u_dict[user_id])
 tflhu[cnt] = timestamp-last_time2_u_dict[user_id]+tfl_hist_u_dict[user_id]*GAMMA
tflhu2[cnt] = timestamp-last_time2_u_dict[user_id]+tfl_hist2_u_dict[user_id]*GAMMA2
if count_up_dict[user_id][part]>0:
tflup[cnt] = last_time_up_dict[user_id][part]-last_time2_up_dict[user_id][part]
tflhup[cnt] = last_time_up_dict[user_id][part]-last_time2_up_dict[user_id][part]+tfl_hist_up_dict[user_id][part]*GAMMA
tflhup2[cnt] = last_time_up_dict[user_id][part]-last_time2_up_dict[user_id][part]+tfl_hist2_up_dict[user_id][part]*GAMMA2
if content_type_id == 0:
acc_std_u_dict[user_id] = update_std(answered_correctly, acc_std_u_dict[user_id], acc_sum_u_dict[user_id], count_u_dict[user_id])
 acc_sum_u_dict[user_id] += answered_correctly
acc_hist_u_dict[user_id] = answered_correctly+acc_hist_u_dict[user_id]*GAMMA
acc_hist2_u_dict[user_id] = answered_correctly+acc_hist2_u_dict[user_id]*GAMMA2
exp_std_u_dict[user_id] = update_std(prior_question_had_explanation, exp_std_u_dict[user_id], exp_sum_u_dict[user_id], count_u_dict[user_id])
 exp_sum_u_dict[user_id] += prior_question_had_explanation
exp_hist_u_dict[user_id] = prior_question_had_explanation+exp_hist_u_dict[user_id]*GAMMA
exp_hist2_u_dict[user_id] = prior_question_had_explanation+exp_hist2_u_dict[user_id]*GAMMA2
time_std_u_dict[user_id] = update_std(prior_question_elapsed_time, time_std_u_dict[user_id], time_sum_u_dict[user_id], count_u_dict[user_id])
 time_sum_u_dict[user_id] += prior_question_elapsed_time
time_hist_u_dict[user_id] = prior_question_elapsed_time+time_hist_u_dict[user_id]*GAMMA
time_hist2_u_dict[user_id] = prior_question_elapsed_time+time_hist2_u_dict[user_id]*GAMMA2
dpt_std_u_dict[user_id] = update_std(( 1-acc_avg_c)*answered_correctly, dpt_std_u_dict[user_id], dpt_sum_u_dict[user_id], count_u_dict[user_id])
 dpt_sum_u_dict[user_id] +=(1-acc_avg_c)*answered_correctly
dpt_hist_u_dict[user_id] =(1-acc_avg_c)*answered_correctly+dpt_hist_u_dict[user_id]*GAMMA
dpt_hist2_u_dict[user_id] =(1-acc_avg_c)*answered_correctly+dpt_hist2_u_dict[user_id]*GAMMA2
dif_std_u_dict[user_id] = update_std(1-acc_avg_c, dif_std_u_dict[user_id], dif_sum_u_dict[user_id], count_u_dict[user_id])
 dif_sum_u_dict[user_id] += 1-acc_avg_c
dif_hist_u_dict[user_id] = 1-acc_avg_c+dif_hist_u_dict[user_id]*GAMMA
dif_hist2_u_dict[user_id] = 1-acc_avg_c+dif_hist2_u_dict[user_id]*GAMMA2
count_u_dict[user_id] += 1
acc_std_c_dict[content_id] = update_std(answered_correctly, acc_std_c_dict[content_id], acc_sum_c_dict[content_id], count_c_dict[content_id])
 acc_sum_c_dict[content_id] += answered_correctly
exp_std_b_dict[prior_bundle] = update_std(prior_question_had_explanation, exp_std_b_dict[prior_bundle], exp_sum_b_dict[prior_bundle], count_b_dict[prior_bundle])
 exp_sum_b_dict[prior_bundle] += prior_question_had_explanation
time_std_b_dict[prior_bundle] = update_std(prior_question_elapsed_time, time_std_b_dict[prior_bundle], time_sum_b_dict[prior_bundle], count_b_dict[prior_bundle])
 time_sum_b_dict[prior_bundle] += prior_question_elapsed_time
count_c_dict[content_id] += 1
count_b_dict[prior_bundle] += 1
acc_sum_up_dict[user_id][part] += answered_correctly
acc_hist_up_dict[user_id][part] = answered_correctly+acc_hist_up_dict[user_id][part]*GAMMA
acc_hist2_up_dict[user_id][part] = answered_correctly+acc_hist2_up_dict[user_id][part]*GAMMA2
count_up_dict[user_id][part] += 1
exp_sum_up_dict[user_id][prior_part] += prior_question_had_explanation
exp_hist_up_dict[user_id][prior_part] = prior_question_had_explanation+exp_hist_up_dict[user_id][prior_part]*GAMMA
exp_hist2_up_dict[user_id][prior_part] = prior_question_had_explanation+exp_hist2_up_dict[user_id][prior_part]*GAMMA2
time_sum_up_dict[user_id][prior_part] += prior_question_elapsed_time
time_hist_up_dict[user_id][prior_part] = prior_question_elapsed_time+time_hist_up_dict[user_id][prior_part]*GAMMA
time_hist2_up_dict[user_id][prior_part] = prior_question_elapsed_time+time_hist2_up_dict[user_id][prior_part]*GAMMA2
count_upp_dict[user_id][prior_part] += 1
acc_sum_uc_dict[user_id][content_id] += answered_correctly
acc_hist_uc_dict[user_id][content_id] = answered_correctly+acc_hist_uc_dict[user_id][content_id]*GAMMA
acc_hist2_uc_dict[user_id][content_id] = answered_correctly+acc_hist2_uc_dict[user_id][content_id]*GAMMA2
count_uc_dict[user_id][content_id] += 1
exp_sum_ub_dict[user_id][prior_bundle] += prior_question_had_explanation
exp_hist_ub_dict[user_id][prior_bundle] = prior_question_had_explanation+exp_hist_ub_dict[user_id][prior_bundle]*GAMMA
exp_hist2_ub_dict[user_id][prior_bundle] = prior_question_had_explanation+exp_hist2_ub_dict[user_id][prior_bundle]*GAMMA2
time_sum_ub_dict[user_id][prior_bundle] += prior_question_elapsed_time
time_hist_ub_dict[user_id][prior_bundle] = prior_question_elapsed_time+time_hist_ub_dict[user_id][prior_bundle]*GAMMA
time_hist2_ub_dict[user_id][prior_bundle] = prior_question_elapsed_time+time_hist2_ub_dict[user_id][prior_bundle]*GAMMA2
count_ub_dict[user_id][prior_bundle] += 1
if last_time_u_dict[user_id] < timestamp:
tfl_std_u_dict[user_id] = update_std(timestamp-last_time_u_dict[user_id], tfl_std_u_dict[user_id], last_time_u_dict[user_id], count_u_dict[user_id]-1)
 tfl_hist_u_dict[user_id] = timestamp-last_time_u_dict[user_id]+tfl_hist_u_dict[user_id]*GAMMA
tfl_hist2_u_dict[user_id] = timestamp-last_time_u_dict[user_id]+tfl_hist2_u_dict[user_id]*GAMMA2
last_time6_u_dict[user_id] = last_time5_u_dict[user_id]
last_time5_u_dict[user_id] = last_time4_u_dict[user_id]
last_time4_u_dict[user_id] = last_time3_u_dict[user_id]
last_time3_u_dict[user_id] = last_time2_u_dict[user_id]
last_time2_u_dict[user_id] = last_time_u_dict[user_id]
last_time_u_dict[user_id] = timestamp
else:
tfl_std_u_dict[user_id] = update_std(timestamp-last_time2_u_dict[user_id], tfl_std_u_dict[user_id], last_time2_u_dict[user_id], count_u_dict[user_id]-1)
 if last_time_up_dict[user_id][part] < timestamp:
tfl_hist_up_dict[user_id][part] = timestamp-last_time_up_dict[user_id][part]+tfl_hist_up_dict[user_id][part]*GAMMA
tfl_hist2_up_dict[user_id][part] = timestamp-last_time_up_dict[user_id][part]+tfl_hist2_up_dict[user_id][part]*GAMMA2
last_time2_up_dict[user_id][part] = last_time_up_dict[user_id][part]
last_time_up_dict[user_id][part] = timestamp
if last_time_uc_dict[user_id][content_id] < timestamp:
tfl_hist_uc_dict[user_id][content_id] = timestamp-last_time_uc_dict[user_id][content_id]+tfl_hist_uc_dict[user_id][content_id]*GAMMA
tfl_hist2_uc_dict[user_id][content_id] = timestamp-last_time_uc_dict[user_id][content_id]+tfl_hist2_uc_dict[user_id][content_id]*GAMMA2
last_time2_uc_dict[user_id][content_id] = last_time_uc_dict[user_id][content_id]
last_time_uc_dict[user_id][content_id] = timestamp
if content_type_id == 1:
count_lecture_u_dict[user_id] += 1
count_lecture_up_dict[user_id][part] += 1
count_lecture_uc_dict[user_id][content_id] += 1
df['acc_sum_u']=acsu
df['acc_std_u']=acstdu
df['acc_hist_u']=achu
df['acc_hist2_u']=achu2
df['exp_sum_u']=exsu
df['exp_std_u']=exstdu
df['exp_hist_u']=exhu
df['exp_hist2_u']=exhu2
df['time_sum_u']=tisu
df['time_std_u']=tistdu
df['time_hist_u']=tihu
df['time_hist2_u']=tihu2
df['dpt_sum_u']=dpsu
df['dpt_std_u']=dpstdu
df['dpt_hist_u']=dphu
df['dpt_hist2_u']=dphu2
df['dif_sum_u']=disu
df['dif_std_u']=disu
df['dif_hist_u']=dihu
df['dif_hist2_u']=dihu2
df['count_u']=cu
df['count_lecture_u']=clu
df['tfl_u']=tflu
df['tfl2_u']=tflu2
df['tfl3_u']=tflu3
df['tfl4_u']=tflu4
df['tfl5_u']=tflu5
df['tfl_std_u']=tflstdu
df['tfl_hist_u']=tflhu
df['tfl_hist2_u']=tflhu2
df['acc_sum_up']=acsup
df['acc_hist_up']=achup
df['acc_hist2_up']=achup2
df['count_up']=cup
df['count_lecture_up']=clup
df['tfl_up']=tflup
df['tfl_hist_up']=tflhup
df['tfl_hist2_up']=tflhup2
df['exp_sum_up']=exsup
df['exp_hist_up']=exhup
df['exp_hist2_up']=exhup2
df['time_sum_up']=tisup
df['time_hist_up']=tihup
df['time_hist2_up']=tihup2
df['count_upp']=cupp
df['count_up1']=cup1
df['count_up2']=cup2
df['count_up3']=cup3
df['count_up4']=cup4
df['count_up5']=cup5
df['count_up6']=cup6
df['count_up7']=cup7
df['acc_sum_uc']=acsuc
df['acc_hist_uc']=achuc
df['acc_hist2_uc']=achuc2
df['count_uc']=cuc
df['count_lecture_uc']=cluc
df['tfl_uc']=tfluc
df['tfl_hist_uc']=tflhuc
df['tfl_hist2_uc']=tflhuc2
df['exp_sum_ub']=exsub
df['exp_hist_ub']=exhub
df['exp_hist2_ub']=exhub2
df['time_sum_ub']=tisub
df['time_hist_ub']=tihub
df['time_hist2_ub']=tihub2
df['count_ub']=cub
df['acc_avg_u'] =(df['acc_sum_u'] / df['count_u'] ).astype('float32')
 df['exp_avg_u'] =(df['exp_sum_u'] / df['count_u'] ).astype('float32')
 df['time_avg_u'] =(df['time_sum_u'] / df['count_u'] ).astype('float32')
 df['dif_avg_u'] =(df['dif_sum_u'] / df['count_u'] ).astype('float32')
 df['dpt_avg_u'] =(df['dpt_sum_u'] / df['count_u'] ).astype('float32')
 df['acc_avg_up'] =(df['acc_sum_up'] / df['count_up'] ).astype('float32')
 df['exp_avg_up'] =(df['exp_sum_up'] / df['count_upp'] ).astype('float32')
 df['time_avg_up'] =(df['time_sum_up'] /df['count_upp'] ).astype('float32')
 df['acc_avg_uc'] =(df['acc_sum_uc'] / df['count_uc'] ).astype('float32')
 df['exp_avg_ub'] =(df['exp_sum_ub'] / df['count_ub'] ).astype('float32')
 df['time_avg_ub'] =(df['time_sum_ub'] /df['count_ub'] ).astype('float32')
 return df
def update_u_db(user_id,acc_sum_u,acc_std_u,acc_hist_u,acc_hist2_u,
exp_sum_u,exp_std_u,exp_hist_u,exp_hist2_u,
time_sum_u,time_std_u,time_hist_u,time_hist2_u,
dpt_sum_u,dpt_std_u,dpt_hist_u,dpt_hist2_u,
dif_sum_u,dif_std_u,dif_hist_u,dif_hist2_u,
count_u,count_lecture_u,
last_time_u,last_time2_u,last_time3_u,last_time4_u,last_time5_u,last_time6_u,
tfl_std_u,tfl_hist_u,tfl_hist2_u,conn):
cur = conn.cursor()
cur.execute("INSERT OR REPLACE INTO u_feats VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",(user_id,acc_sum_u,acc_std_u,acc_hist_u,acc_hist2_u,exp_sum_u,exp_std_u,exp_hist_u,exp_hist2_u,time_sum_u,time_std_u,time_hist_u,time_hist2_u,dpt_sum_u,dpt_std_u,dpt_hist_u,dpt_hist2_u,dif_sum_u,dif_std_u,dif_hist_u,dif_hist2_u,count_u,count_lecture_u,last_time_u,last_time2_u,last_time3_u,last_time4_u,last_time5_u,last_time6_u,tfl_std_u,tfl_hist_u,tfl_hist2_u))
cur.close()
def update_c_db(content_id,acc_sum_c,acc_std_c,count_c,conn):
cur = conn.cursor()
cur.execute("INSERT OR REPLACE INTO c_feats VALUES(?,?,?,?)",(content_id,acc_sum_c,acc_std_c,count_c))
cur.close()
def update_b_db(bundle_id,exp_sum_b,exp_std_b,time_sum_b,time_std_b,count_b,conn):
cur = conn.cursor()
cur.execute("INSERT OR REPLACE INTO b_feats VALUES(?,?,?,?,?,?)",(bundle_id,exp_sum_b,exp_std_b,time_sum_b,time_std_b,count_b))
cur.close()
def update_up_db(user_id,part,
acc_sum_up,acc_hist_up,acc_hist2_up,
count_up,count_lecture_up,last_time_up,
last_time2_up,tfl_hist_up,tfl_hist2_up,
exp_sum_up,exp_hist_up,exp_hist2_up,
time_sum_up,time_hist_up,time_hist2_up,count_upp,conn):
cur = conn.cursor()
cur.execute("INSERT OR REPLACE INTO up_feats VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",(user_id,part,acc_sum_up,acc_hist_up,acc_hist2_up, count_up,count_lecture_up,last_time_up, last_time2_up,tfl_hist_up,tfl_hist2_up, exp_sum_up,exp_hist_up,exp_hist2_up,time_sum_up,time_hist_up,time_hist2_up,count_upp))
cur.close()
def update_uc_db(user_id,content_id,
acc_sum_uc,acc_hist_uc,acc_hist2_uc,
count_uc,count_lecture_uc,last_time_uc,
last_time2_uc,tfl_hist_uc,tfl_hist2_uc,conn):
cur = conn.cursor()
cur.execute("INSERT OR REPLACE INTO uc_feats VALUES(?,?,?,?,?,?,?,?,?,?,?)",(user_id,content_id,acc_sum_uc, acc_hist_uc, acc_hist2_uc,count_uc,count_lecture_uc,last_time_uc,last_time2_uc,tfl_hist_uc,tfl_hist2_uc))
cur.close()
def update_ub_db(user_id,bundle_id,
exp_sum_ub,exp_hist_ub,exp_hist2_ub,
time_sum_ub,time_hist_ub,time_hist2_ub,count_ub,conn):
cur = conn.cursor()
cur.execute("INSERT OR REPLACE INTO ub_feats VALUES(?,?,?,?,?,?,?,?,?)",(user_id,bundle_id,exp_sum_ub,exp_hist_ub, exp_hist2_ub,time_sum_ub,time_hist_ub,time_hist2_ub,count_ub))
cur.close()
def get_u_feats(user_id,conn):
cur = conn.cursor()
cur.execute("SELECT * FROM u_feats WHERE user_id = "+str(user_id))
list_u=cur.fetchone()
cur.close()
if list_u==None:
acc_sum_u=0
acc_std_u=0
acc_hist_u=0
acc_hist2_u=0
exp_sum_u=0
exp_std_u=0
exp_hist_u=0
exp_hist2_u=0
time_sum_u=0
time_std_u=0
time_hist_u=0
time_hist2_u=0
dpt_sum_u=0
dpt_std_u=0
dpt_hist_u=0
dpt_hist2_u=0
dif_sum_u=0
dif_std_u=0
dif_hist_u=0
dif_hist2_u=0
count_u=0
count_lecture_u=0
last_time_u=0
last_time2_u=0
last_time3_u=0
last_time4_u=0
last_time5_u=0
last_time6_u=0
tfl_std_u=0
tfl_hist_u=0
tfl_hist2_u=0
else:
acc_sum_u=list_u[1]
acc_std_u=list_u[2]
acc_hist_u=list_u[3]
acc_hist2_u=list_u[4]
exp_sum_u=list_u[5]
exp_std_u=list_u[6]
exp_hist_u=list_u[7]
exp_hist2_u=list_u[8]
time_sum_u=list_u[9]
time_std_u=list_u[10]
time_hist_u=list_u[11]
time_hist2_u=list_u[12]
dpt_sum_u=list_u[13]
dpt_std_u=list_u[14]
dpt_hist_u=list_u[15]
dpt_hist2_u=list_u[16]
dif_sum_u=list_u[17]
dif_std_u=list_u[18]
dif_hist_u=list_u[19]
dif_hist2_u=list_u[20]
count_u=list_u[21]
count_lecture_u=list_u[22]
last_time_u=list_u[23]
last_time2_u=list_u[24]
last_time3_u=list_u[25]
last_time4_u=list_u[26]
last_time5_u=list_u[27]
last_time6_u=list_u[28]
tfl_std_u=list_u[29]
tfl_hist_u=list_u[30]
tfl_hist2_u=list_u[31]
return acc_sum_u,acc_std_u,acc_hist_u,acc_hist2_u,exp_sum_u,exp_std_u,exp_hist_u,exp_hist2_u, time_sum_u,time_std_u,time_hist_u,time_hist2_u,dpt_sum_u,dpt_std_u,dpt_hist_u,dpt_hist2_u, dif_sum_u,dif_std_u,dif_hist_u,dif_hist2_u, count_u,count_lecture_u,last_time_u,last_time2_u,last_time3_u,last_time4_u,last_time5_u,last_time6_u,tfl_std_u,tfl_hist_u,tfl_hist2_u
def get_c_feats(content_id,conn):
cur = conn.cursor()
cur.execute("SELECT * FROM c_feats WHERE content_id = "+str(content_id))
list_c=cur.fetchone()
cur.close()
if list_c==None:
acc_sum_c=0
acc_std_c=0
count_c=0
else:
acc_sum_c=list_c[1]
acc_std_c=list_c[2]
count_c=list_c[3]
return acc_sum_c,acc_std_c,count_c
def get_b_feats(bundle_id,conn):
cur = conn.cursor()
cur.execute("SELECT * FROM b_feats WHERE bundle_id = "+str(bundle_id))
list_b=cur.fetchone()
cur.close()
if list_b==None:
exp_sum_b=0
exp_std_b=0
time_sum_b=0
time_std_b=0
count_b=0
else:
exp_sum_b=list_b[1]
exp_std_b=list_b[2]
time_sum_b=list_b[3]
time_std_b=list_b[4]
count_b=list_b[5]
return exp_sum_b,exp_std_b,time_sum_b,time_std_b,count_b
def get_up_feats(user_id,part,conn):
cur = conn.cursor()
cur.execute("SELECT * FROM up_feats WHERE user_id = "+str(user_id)+' AND part = '+str(part))
list_up=cur.fetchone()
cur.close()
if list_up==None:
acc_sum_up=0
acc_hist_up=0
acc_hist2_up=0
count_up=0
count_lecture_up=0
last_time_up=0
last_time2_up=0
tfl_hist_up=0
tfl_hist2_up=0
exp_sum_up=0
exp_hist_up=0
exp_hist2_up=0
time_sum_up=0
time_hist_up=0
time_hist2_up=0
count_upp=0
else:
acc_sum_up=list_up[2]
acc_hist_up=list_up[3]
acc_hist2_up=list_up[4]
count_up=list_up[5]
count_lecture_up=list_up[6]
last_time_up=list_up[7]
last_time2_up=list_up[8]
tfl_hist_up=list_up[9]
tfl_hist2_up=list_up[10]
exp_sum_up=list_up[11]
exp_hist_up=list_up[12]
exp_hist2_up=list_up[13]
time_sum_up=list_up[14]
time_hist_up=list_up[15]
time_hist2_up=list_up[16]
count_upp=list_up[17]
return acc_sum_up,acc_hist_up,acc_hist2_up,count_up,count_lecture_up,last_time_up,last_time2_up,tfl_hist_up,tfl_hist2_up,exp_sum_up,exp_hist_up,exp_hist2_up,time_sum_up,time_hist_up,time_hist2_up,count_upp
def get_up_counts(user_id,conn):
cur = conn.cursor()
cur.execute("SELECT * FROM up_feats WHERE user_id = "+str(user_id)+' AND part = 1')
 list_up1=cur.fetchone()
cur.execute("SELECT * FROM up_feats WHERE user_id = "+str(user_id)+' AND part = 2')
 list_up2=cur.fetchone()
cur.execute("SELECT * FROM up_feats WHERE user_id = "+str(user_id)+' AND part = 3')
 list_up3=cur.fetchone()
cur.execute("SELECT * FROM up_feats WHERE user_id = "+str(user_id)+' AND part = 4')
 list_up4=cur.fetchone()
cur.execute("SELECT * FROM up_feats WHERE user_id = "+str(user_id)+' AND part = 5')
 list_up5=cur.fetchone()
cur.execute("SELECT * FROM up_feats WHERE user_id = "+str(user_id)+' AND part = 6')
 list_up6=cur.fetchone()
cur.execute("SELECT * FROM up_feats WHERE user_id = "+str(user_id)+' AND part = 7')
 list_up7=cur.fetchone()
cur.close()
if list_up1==None:
count_up1=0
else:
count_up1=list_up1[5]
if list_up2==None:
count_up2=0
else:
count_up2=list_up2[5]
if list_up3==None:
count_up3=0
else:
count_up3=list_up3[5]
if list_up4==None:
count_up4=0
else:
count_up4=list_up4[5]
if list_up5==None:
count_up5=0
else:
count_up5=list_up5[5]
if list_up6==None:
count_up6=0
else:
count_up6=list_up6[5]
if list_up7==None:
count_up7=0
else:
count_up7=list_up7[5]
return count_up1,count_up2,count_up3,count_up4,count_up5,count_up6,count_up7
def get_uc_feats(user_id,content_id,conn):
cur = conn.cursor()
cur.execute("SELECT * FROM uc_feats WHERE user_id = "+str(user_id)+' AND content_id = '+str(content_id))
list_uc=cur.fetchone()
cur.close()
if list_uc==None:
acc_sum_uc=0
acc_hist_uc=0
acc_hist2_uc=0
count_uc=0
count_lecture_uc=0
last_time_uc=0
last_time2_uc=0
tfl_hist_uc=0
tfl_hist2_uc=0
else:
acc_sum_uc=list_uc[2]
acc_hist_uc=list_uc[3]
acc_hist2_uc=list_uc[4]
count_uc=list_uc[5]
count_lecture_uc=list_uc[6]
last_time_uc=list_uc[7]
last_time2_uc=list_uc[8]
tfl_hist_uc=list_uc[9]
tfl_hist2_uc=list_uc[10]
return acc_sum_uc,acc_hist_uc,acc_hist2_uc,count_uc,count_lecture_uc,last_time_uc,last_time2_uc,tfl_hist_uc,tfl_hist2_uc
def get_ub_feats(user_id,bundle_id,conn):
cur = conn.cursor()
cur.execute("SELECT * FROM ub_feats WHERE user_id = "+str(user_id)+' AND bundle_id = '+str(bundle_id))
list_ub=cur.fetchone()
cur.close()
if list_ub==None:
exp_sum_ub=0
exp_hist_ub=0
exp_hist2_ub=0
time_sum_ub=0
time_hist_ub=0
time_hist2_ub=0
count_ub=0
else:
exp_sum_ub=list_ub[2]
exp_hist_ub=list_ub[3]
exp_hist2_ub=list_ub[4]
time_sum_ub=list_ub[5]
time_hist_ub=list_ub[6]
time_hist2_ub=list_ub[7]
count_ub=list_ub[8]
return exp_sum_ub,exp_hist_ub,exp_hist2_ub,time_sum_ub,time_hist_ub,time_hist2_ub,count_ub
def add_feats_without_update_sqlite(df, conn):
acsu = np.zeros(len(df), dtype=np.int16)
 acstdu = np.zeros(len(df), dtype=np.float32)
 achu = np.zeros(len(df), dtype=np.float32)
 achu2 = np.zeros(len(df), dtype=np.float32)
 exsu = np.zeros(len(df), dtype=np.int16)
 exstdu = np.zeros(len(df), dtype=np.float32)
 exhu = np.zeros(len(df), dtype=np.float32)
 exhu2 = np.zeros(len(df), dtype=np.float32)
 tisu = np.zeros(len(df), dtype=np.int32)
 tistdu = np.zeros(len(df), dtype=np.float32)
 tihu = np.zeros(len(df), dtype=np.float32)
 tihu2 = np.zeros(len(df), dtype=np.float32)
 dpsu = np.zeros(len(df), dtype=np.float32)
 dpstdu = np.zeros(len(df), dtype=np.float32)
 dphu = np.zeros(len(df), dtype=np.float32)
 dphu2 = np.zeros(len(df), dtype=np.float32)
 disu = np.zeros(len(df), dtype=np.float32)
 distdu = np.zeros(len(df), dtype=np.float32)
 dihu = np.zeros(len(df), dtype=np.float32)
 dihu2 = np.zeros(len(df), dtype=np.float32)
 cu = np.zeros(len(df), dtype=np.int16)
 clu = np.zeros(len(df), dtype=np.int16)
 tflu = np.zeros(len(df), dtype=np.int32)
 tflu2 = np.zeros(len(df), dtype=np.int32)
 tflu3 = np.zeros(len(df), dtype=np.int32)
 tflu4 = np.zeros(len(df), dtype=np.int32)
 tflu5 = np.zeros(len(df), dtype=np.int32)
 tflstdu = np.zeros(len(df), dtype=np.float32)
 tflhu = np.zeros(len(df), dtype=np.int32)
 tflhu2 = np.zeros(len(df), dtype=np.int32)
 acsc = np.zeros(len(df), dtype=np.int32)
 exsb = np.zeros(len(df), dtype=np.int32)
 tisb = np.zeros(len(df), dtype=np.int32)
 cc = np.zeros(len(df), dtype=np.int32)
 cb = np.zeros(len(df), dtype=np.int32)
 acsup = np.zeros(len(df), dtype=np.int16)
 achup = np.zeros(len(df), dtype=np.float32)
 achup2 = np.zeros(len(df), dtype=np.float32)
 cup = np.zeros(len(df), dtype=np.int16)
 clup = np.zeros(len(df), dtype=np.int16)
 tflup = np.zeros(len(df), dtype=np.int32)
 tflhup = np.zeros(len(df), dtype=np.int32)
 tflhup2 = np.zeros(len(df), dtype=np.int32)
 exsup = np.zeros(len(df), dtype=np.int16)
 exhup = np.zeros(len(df), dtype=np.float32)
 exhup2 = np.zeros(len(df), dtype=np.float32)
 tisup = np.zeros(len(df), dtype=np.int32)
 tihup = np.zeros(len(df), dtype=np.float32)
 tihup2 = np.zeros(len(df), dtype=np.float32)
 cupp = np.zeros(len(df), dtype=np.int32)
 cup1 = np.zeros(len(df), dtype=np.int16)
 cup2 = np.zeros(len(df), dtype=np.int16)
 cup3 = np.zeros(len(df), dtype=np.int16)
 cup4 = np.zeros(len(df), dtype=np.int16)
 cup5 = np.zeros(len(df), dtype=np.int16)
 cup6 = np.zeros(len(df), dtype=np.int16)
 cup7 = np.zeros(len(df), dtype=np.int16)
 acsuc = np.zeros(len(df), dtype=np.int16)
 achuc = np.zeros(len(df), dtype=np.float32)
 achuc2 = np.zeros(len(df), dtype=np.float32)
 cuc = np.zeros(len(df), dtype=np.int16)
 tfluc = np.zeros(len(df), dtype=np.int32)
 tflhuc = np.zeros(len(df), dtype=np.int32)
 tflhuc2 = np.zeros(len(df), dtype=np.int32)
 exsub = np.zeros(len(df), dtype=np.int32)
 exhub = np.zeros(len(df), dtype=np.float32)
 exhub2 = np.zeros(len(df), dtype=np.float32)
 tisub = np.zeros(len(df), dtype=np.int32)
 tihub = np.zeros(len(df), dtype=np.float32)
 tihub2 = np.zeros(len(df), dtype=np.float32)
 cub = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(df[['content_type_id',
'content_type_id',
'prior_question_had_explanation',
'prior_question_elapsed_time',
'acc_avg_c',
'user_id',
'part',
'prior_part',
'content_id',
'bundle_id',
'prior_bundle',
'timestamp']].values):
content_type_id=int(row[0])
 prior_question_had_explanation=int(row[2])
 prior_question_elapsed_time=int(row[3])
 acc_avg_c=float(row[4])
 user_id=int(row[5])
 part=int(row[6])
 prior_part=int(row[7])
 content_id=int(row[8])
 bundle_id=int(row[9])
 prior_bundle=int(row[10])
 timestamp=int(row[11])
 acc_sum_u,acc_std_u,acc_hist_u,acc_hist2_u,exp_sum_u,exp_std_u,exp_hist_u,exp_hist2_u, time_sum_u,time_std_u,time_hist_u,time_hist2_u,dpt_sum_u,dpt_std_u,dpt_hist_u,dpt_hist2_u,dif_sum_u,dif_std_u,dif_hist_u,dif_hist2_u, count_u,count_lecture_u,last_time_u,last_time2_u,last_time3_u,last_time4_u,last_time5_u,last_time6_u,tfl_std_u,tfl_hist_u,tfl_hist2_u=get_u_feats(user_id,conn)
 acc_sum_c,acc_std_c,count_c=get_c_feats(content_id,conn)
 exp_sum_b,exp_std_b,time_sum_b,time_std_b,count_b=get_b_feats(bundle_id,conn)
 acc_sum_up,acc_hist_up,acc_hist2_up,count_up,count_lecture_up,last_time_up,last_time2_up,tfl_hist_up,tfl_hist2_up,exp_sum_up,exp_hist_up,exp_hist2_up,time_sum_up,time_hist_up,time_hist2_up,count_upp=get_up_feats(user_id,part,conn)
 acc_sum_uc,acc_hist_uc,acc_hist2_uc,count_uc,count_lecture_uc,last_time_uc,last_time2_uc,tfl_hist_uc,tfl_hist2_uc=get_uc_feats(user_id,content_id,conn)
 exp_sum_ub,exp_hist_ub,exp_hist2_ub,time_sum_ub,time_hist_ub,time_hist2_ub,count_ub=get_ub_feats(user_id,bundle_id,conn)
 count_up1,count_up2,count_up3,count_up4,count_up5,count_up6,count_up7=get_up_counts(user_id,conn)
 acsu[cnt] = acc_sum_u
acstdu[cnt] = acc_std_u
achu[cnt] = acc_hist_u
achu2[cnt] = acc_hist2_u
exsu[cnt] = exp_sum_u
exstdu[cnt] = exp_std_u
exhu[cnt] = exp_hist_u
exhu2[cnt] = exp_hist2_u
tisu[cnt] = time_sum_u
tistdu[cnt] = time_std_u
tihu[cnt] = time_hist_u
tihu2[cnt] = time_hist2_u
dpsu[cnt] = dpt_sum_u
dpstdu[cnt] = dpt_std_u
dphu[cnt] = dpt_hist_u
dphu2[cnt] = dpt_hist2_u
disu[cnt] = dif_sum_u
distdu[cnt] = dif_std_u
dihu[cnt] = dif_hist_u
dihu2[cnt] = dif_hist2_u
cu[cnt] = count_u
clu[cnt] = count_lecture_u
tflu[cnt] = timestamp-last_time_u
tflu2[cnt] = last_time_u-last_time2_u
tflu3[cnt] = last_time2_u-last_time3_u
tflu4[cnt] = last_time3_u-last_time4_u
tflu5[cnt] = last_time4_u-last_time5_u
tflstdu[cnt] = update_std(timestamp-last_time_u, tfl_std_u, last_time_u, count_u)
 tflhu[cnt] = timestamp-last_time_u+tfl_hist_u*GAMMA
tflhu2[cnt] = timestamp-last_time_u+tfl_hist2_u*GAMMA2
acsc[cnt] = acc_sum_c
exsb[cnt] = exp_sum_b
tisb[cnt] = time_sum_b
cc[cnt] = count_c
cb[cnt] = count_b
acsup[cnt] = acc_sum_up
achup[cnt] = acc_hist_up
achup2[cnt] = acc_hist2_up
cup[cnt] = count_up
clup[cnt] = count_lecture_up
if count_up>0:
tflup[cnt] = timestamp-last_time_up
tflhup[cnt] = timestamp-last_time_up+tfl_hist_up*GAMMA
tflhup2[cnt] = timestamp-last_time_up+tfl_hist2_up*GAMMA2
exsup[cnt] = exp_sum_up
exhup[cnt] = exp_hist_up
exhup2[cnt] = exp_hist2_up
tisup[cnt] = time_sum_up
tihup[cnt] = time_hist_up
tihup2[cnt] = time_hist2_up
cupp[cnt] = count_upp
cup1[cnt] = count_up1
cup2[cnt] = count_up2
cup3[cnt] = count_up3
cup4[cnt] = count_up4
cup5[cnt] = count_up5
cup6[cnt] = count_up6
cup7[cnt] = count_up7
acsuc[cnt] = acc_sum_uc
achuc[cnt] = acc_hist_uc
achuc2[cnt] = acc_hist2_uc
cuc[cnt] = count_uc
if count_uc>0:
tfluc[cnt] = timestamp-last_time_uc
tflhuc[cnt] = timestamp-last_time_uc+tfl_hist_uc*GAMMA
tflhuc2[cnt] = timestamp-last_time_uc+tfl_hist2_uc*GAMMA2
exsub[cnt] = exp_sum_ub
exhub[cnt] = exp_hist_ub
exhub2[cnt] = exp_hist2_ub
tisub[cnt] = time_sum_ub
tihub[cnt] = time_hist_ub
tihub2[cnt] = time_hist2_ub
cub[cnt] = count_ub
if last_time_u == timestamp:
tflu[cnt] = timestamp-last_time2_u
tflu2[cnt] = last_time2_u-last_time3_u
tflu3[cnt] = last_time3_u-last_time4_u
tflu4[cnt] = last_time4_u-last_time5_u
tflu5[cnt] = last_time5_u-last_time6_u
tflstdu[cnt] = update_std(timestamp-last_time2_u, tfl_std_u, last_time2_u, count_u)
 tflhu[cnt] = timestamp-last_time2_u+tfl_hist_u*GAMMA
tflhu2[cnt] = timestamp-last_time2_u+tfl_hist2_u*GAMMA2
if count_up>0:
tflup[cnt] = last_time_up-last_time2_up
tflhup[cnt] = last_time_up-last_time2_up+tfl_hist_up*GAMMA
tflhup2[cnt] = last_time_up-last_time2_up+tfl_hist2_up*GAMMA2
df['acc_sum_u']=acsu
df['acc_std_u']=acstdu
df['acc_hist_u']=achu
df['acc_hist2_u']=achu2
df['exp_sum_u']=exsu
df['exp_std_u']=exstdu
df['exp_hist_u']=exhu
df['exp_hist2_u']=exhu2
df['time_sum_u']=tisu
df['time_std_u']=tistdu
df['time_hist_u']=tihu
df['time_hist2_u']=tihu2
df['dpt_sum_u']=dpsu
df['dpt_std_u']=dpsu
df['dpt_hist_u']=dphu
df['dpt_hist2_u']=dphu2
df['dif_sum_u']=disu
df['dif_std_u']=disu
df['dif_hist_u']=dihu
df['dif_hist2_u']=dihu2
df['count_u']=cu
df['count_lecture_u']=clu
df['tfl_u']=tflu
df['tfl2_u']=tflu2
df['tfl3_u']=tflu3
df['tfl4_u']=tflu4
df['tfl5_u']=tflu5
df['tfl_std_u']=tflstdu
df['tfl_hist_u']=tflhu
df['tfl_hist2_u']=tflhu2
df['acc_sum_c']=acsc
df['exp_sum_b']=exsb
df['time_sum_b']=tisb
df['count_c']=cc
df['count_b']=cb
df['acc_sum_up']=acsup
df['acc_hist_up']=achup
df['acc_hist2_up']=achup2
df['count_up']=cup
df['count_lecture_up']=clup
df['tfl_up']=tflup
df['tfl_hist_up']=tflhup
df['tfl_hist2_up']=tflhup2
df['exp_sum_up']=exsup
df['exp_hist_up']=exhup
df['exp_hist2_up']=exhup2
df['time_sum_up']=tisup
df['time_hist_up']=tihup
df['time_hist2_up']=tihup2
df['count_upp']=cupp
df['count_up1']=cup1
df['count_up2']=cup2
df['count_up3']=cup3
df['count_up4']=cup4
df['count_up5']=cup5
df['count_up6']=cup6
df['count_up7']=cup7
df['acc_sum_uc']=acsuc
df['acc_hist_uc']=achuc
df['acc_hist2_uc']=achuc2
df['count_uc']=cuc
df['tfl_uc']=tfluc
df['tfl_hist_uc']=tflhuc
df['tfl_hist2_uc']=tflhuc2
df['exp_sum_ub']=exsub
df['exp_hist_ub']=exhub
df['exp_hist2_ub']=exhub2
df['time_sum_ub']=tisub
df['time_hist_ub']=tihub
df['time_hist2_ub']=tihub2
df['count_ub']=cub
df['acc_avg_u'] =(df['acc_sum_u'] / df['count_u'] ).astype('float32')
 df['exp_avg_u'] =(df['exp_sum_u'] / df['count_u'] ).astype('float32')
 df['time_avg_u'] =(df['time_sum_u'] / df['count_u'] ).astype('float32')
 df['dif_avg_u'] =(df['dif_sum_u'] / df['count_u'] ).astype('float32')
 df['dpt_avg_u'] =(df['dpt_sum_u'] / df['count_u'] ).astype('float32')
 df['acc_avg_up'] =(df['acc_sum_up'] / df['count_up'] ).astype('float32')
 df['exp_avg_up'] =(df['exp_sum_up'] / df['count_upp'] ).astype('float32')
 df['time_avg_up'] =(df['time_sum_up'] /df['count_upp'] ).astype('float32')
 df['acc_avg_uc'] =(df['acc_sum_uc'] / df['count_uc'] ).astype('float32')
 df['exp_avg_ub'] =(df['exp_sum_ub'] / df['count_ub'] ).astype('float32')
 df['time_avg_ub'] =(df['time_sum_ub'] /df['count_ub'] ).astype('float32')
 return df
def update_feats_sqlite(df, conn):
for cnt,row in enumerate(df[['content_type_id',
'answered_correctly',
'prior_question_had_explanation',
'prior_question_elapsed_time',
'acc_avg_c',
'user_id',
'part',
'prior_part',
'content_id',
'bundle_id',
'prior_bundle',
'timestamp']].values):
content_type_id=int(row[0])
 answered_correctly=int(row[1])
 prior_question_had_explanation=int(row[2])
 prior_question_elapsed_time=int(row[3])
 acc_avg_c=float(row[4])
 user_id=int(row[5])
 part=int(row[6])
 prior_part=int(row[7])
 content_id=int(row[8])
 bundle_id=int(row[9])
 prior_bundle=int(row[10])
 timestamp=int(row[11])
 acc_sum_u,acc_std_u,acc_hist_u,acc_hist2_u,exp_sum_u,exp_std_u,exp_hist_u,exp_hist2_u, time_sum_u,time_std_u,time_hist_u,time_hist2_u,dpt_sum_u,dpt_std_u,dpt_hist_u,dpt_hist2_u,dif_sum_u,dif_std_u,dif_hist_u,dif_hist2_u, count_u,count_lecture_u,last_time_u,last_time2_u,last_time3_u,last_time4_u,last_time5_u,last_time6_u,tfl_std_u,tfl_hist_u,tfl_hist2_u=get_u_feats(user_id,conn)
 acc_sum_c,acc_std_c,count_c=get_c_feats(content_id,conn)
 exp_sum_b,exp_std_b,time_sum_b,time_std_b,count_b=get_b_feats(bundle_id,conn)
 acc_sum_up,acc_hist_up,acc_hist2_up,count_up,count_lecture_up,last_time_up,last_time2_up,tfl_hist_up,tfl_hist2_up,exp_sum_up,exp_hist_up,exp_hist2_up,time_sum_up,time_hist_up,time_hist2_up,count_upp=get_up_feats(user_id,part,conn)
 acc_sum_uc,acc_hist_uc,acc_hist2_uc,count_uc,count_lecture_uc,last_time_uc,last_time2_uc,tfl_hist_uc,tfl_hist2_uc=get_uc_feats(user_id,content_id,conn)
 exp_sum_ub,exp_hist_ub,exp_hist2_ub,time_sum_ub,time_hist_ub,time_hist2_ub,count_ub=get_ub_feats(user_id,bundle_id,conn)
 count_up1,count_up2,count_up3,count_up4,count_up5,count_up6,count_up7=get_up_counts(user_id,conn)
 if content_type_id == 0:
acc_std_u = update_std(answered_correctly, acc_std_u, acc_sum_u, count_u)
 acc_sum_u += answered_correctly
acc_hist_u = answered_correctly+acc_hist_u*GAMMA
acc_hist2_u = answered_correctly+acc_hist2_u*GAMMA2
exp_std_u = update_std(prior_question_had_explanation, exp_std_u, exp_sum_u, count_u)
 exp_sum_u += prior_question_had_explanation
exp_hist_u = prior_question_had_explanation+exp_hist_u*GAMMA
exp_hist2_u = prior_question_had_explanation+exp_hist2_u*GAMMA2
time_std_u = update_std(prior_question_elapsed_time, time_std_u, time_sum_u, count_u)
 time_sum_u += prior_question_elapsed_time
time_hist_u = prior_question_elapsed_time+time_hist_u*GAMMA
time_hist2_u = prior_question_elapsed_time+time_hist2_u*GAMMA2
dpt_std_u = update_std(( 1-acc_avg_c)*answered_correctly, dpt_std_u, dpt_sum_u, count_u)
 dpt_sum_u +=(1-acc_avg_c)*answered_correctly
dpt_hist_u =(1-acc_avg_c)*answered_correctly+dpt_hist_u*GAMMA
dpt_hist2_u =(1-acc_avg_c)*answered_correctly+dpt_hist2_u*GAMMA2
dif_std_u = update_std(1-acc_avg_c, dif_std_u, dif_sum_u, count_u)
 dif_sum_u += 1-acc_avg_c
dif_hist_u = 1-acc_avg_c+dif_hist_u*GAMMA
dif_hist2_u = 1-acc_avg_c+dif_hist2_u*GAMMA2
count_u += 1
acc_sum_c += answered_correctly
exp_sum_b += prior_question_had_explanation
time_sum_b += prior_question_elapsed_time
count_c += 1
count_b += 1
acc_sum_up += answered_correctly
acc_hist_up = answered_correctly+acc_hist_up*GAMMA
acc_hist2_up = answered_correctly+acc_hist2_up*GAMMA2
count_up += 1
exp_sum_up += prior_question_had_explanation
exp_hist_up = prior_question_had_explanation+exp_hist_up*GAMMA
exp_hist2_up = prior_question_had_explanation+exp_hist2_up*GAMMA2
time_sum_up += prior_question_elapsed_time
time_hist_up = prior_question_elapsed_time+time_hist_up*GAMMA
time_hist2_up = prior_question_elapsed_time+time_hist2_up*GAMMA2
count_upp += 1
acc_sum_uc += answered_correctly
acc_hist_uc = answered_correctly+acc_hist_uc*GAMMA
acc_hist2_uc = answered_correctly+acc_hist2_uc*GAMMA2
count_uc += 1
exp_sum_ub += prior_question_had_explanation
exp_hist_ub = prior_question_had_explanation+exp_hist_ub*GAMMA
exp_hist2_ub = prior_question_had_explanation+exp_hist2_ub*GAMMA2
time_sum_ub += prior_question_elapsed_time
time_hist_ub = prior_question_elapsed_time+time_hist_ub*GAMMA
time_hist2_ub = prior_question_elapsed_time+time_hist2_ub*GAMMA2
count_ub += 1
if last_time_u < timestamp:
tfl_std_u = update_std(timestamp-last_time_u, tfl_std_u, last_time_u, count_u-1)
 tfl_hist_u = timestamp-last_time_u+tfl_hist_u*GAMMA
tfl_hist2_u = timestamp-last_time_u+tfl_hist2_u*GAMMA2
last_time6_u = last_time5_u
last_time5_u = last_time4_u
last_time4_u = last_time3_u
last_time3_u = last_time2_u
last_time2_u = last_time_u
last_time_u = timestamp
else:
tfl_std_u = update_std(timestamp-last_time2_u, tfl_std_u, last_time2_u, count_u-1)
 if last_time_up < timestamp:
tfl_hist_up = timestamp-last_time_up+tfl_hist_up*GAMMA
tfl_hist2_up = timestamp-last_time_up+tfl_hist2_up*GAMMA2
last_time2_up = last_time_up
last_time_up = timestamp
if last_time_uc < timestamp:
tfl_hist_uc = timestamp-last_time_uc+tfl_hist_uc*GAMMA
tfl_hist2_uc = timestamp-last_time_uc+tfl_hist2_uc*GAMMA2
last_time2_uc = last_time_uc
last_time_uc = timestamp
update_u_db(user_id,acc_sum_u,acc_std_u,acc_hist_u,acc_hist2_u,
exp_sum_u,exp_std_u,exp_hist_u,exp_hist2_u,
time_sum_u,time_std_u,time_hist_u,time_hist2_u,
dpt_sum_u,dpt_std_u,dpt_hist_u,dpt_hist2_u,
dif_sum_u,dif_std_u,dif_hist_u,dif_hist2_u,
count_u,count_lecture_u,
last_time_u,last_time2_u,last_time3_u,last_time4_u,last_time5_u,last_time6_u,
tfl_std_u,tfl_hist_u,tfl_hist2_u,conn)
 update_c_db(content_id,acc_sum_c,acc_std_c,count_c,conn)
 update_b_db(bundle_id,exp_sum_b,exp_std_b,time_sum_b,time_std_b,count_b,conn)
 update_up_db(user_id,part,
acc_sum_up,acc_hist_up,acc_hist2_up,
count_up,count_lecture_up,last_time_up,
last_time2_up,tfl_hist_up,tfl_hist2_up,
exp_sum_up,exp_hist_up,exp_hist2_up,
time_sum_up,time_hist_up,time_hist2_up,count_upp,conn)
 update_uc_db(user_id,content_id,
acc_sum_uc,acc_hist_uc,acc_hist2_uc,
count_uc,count_lecture_uc,last_time_uc,
last_time2_uc,tfl_hist_uc,tfl_hist2_uc,conn)
 update_ub_db(user_id,bundle_id,
exp_sum_ub,exp_hist_ub,exp_hist2_ub,
time_sum_ub,time_hist_ub,time_hist2_ub,count_ub,conn)
 if content_type_id == 1:
count_lecture_u += 1
count_lecture_up += 1
update_u_db(user_id,acc_sum_u,acc_std_u,acc_hist_u,acc_hist2_u,
exp_sum_u,exp_std_u,exp_hist_u,exp_hist2_u,
time_sum_u,time_std_u,time_hist_u,time_hist2_u,
dpt_sum_u,dpt_std_u,dpt_hist_u,dpt_hist2_u,
dif_sum_u,dif_std_u,dif_hist_u,dif_hist2_u,
count_u,count_lecture_u,
last_time_u,last_time2_u,last_time3_u,last_time4_u,last_time5_u,last_time6_u,
tfl_std_u,tfl_hist_u,tfl_hist2_u,conn)
 update_up_db(user_id,part,
acc_sum_up,acc_hist_up,acc_hist2_up,
count_up,count_lecture_up,last_time_up,
last_time2_up,tfl_hist_up,tfl_hist2_up,
exp_sum_up,exp_hist_up,exp_hist2_up,
time_sum_up,time_hist_up,time_hist2_up,count_upp,conn )<categorify>
train[target_cols] = oof
train[['sig_id']+target_cols].to_csv('oof.csv', index=False)
 test[target_cols] = predictions
test[['sig_id']+target_cols].to_csv('pred.csv', index=False )
acc_sum_u_dict = defaultdict(int)
 acc_std_u_dict = defaultdict(float)
 acc_hist_u_dict = defaultdict(float)
 acc_hist2_u_dict = defaultdict(float)
 exp_sum_u_dict = defaultdict(int)
 exp_std_u_dict = defaultdict(float)
 exp_hist_u_dict = defaultdict(float)
 exp_hist2_u_dict = defaultdict(float)
 time_sum_u_dict = defaultdict(int)
 time_std_u_dict = defaultdict(float)
 time_hist_u_dict = defaultdict(float)
 time_hist2_u_dict = defaultdict(float)
 dpt_sum_u_dict = defaultdict(float)
 dpt_std_u_dict = defaultdict(float)
 dpt_hist_u_dict = defaultdict(float)
 dpt_hist2_u_dict = defaultdict(float)
 dif_sum_u_dict = defaultdict(float)
 dif_std_u_dict = defaultdict(float)
 dif_hist_u_dict = defaultdict(float)
 dif_hist2_u_dict = defaultdict(float)
 count_u_dict = defaultdict(int)
 count_lecture_u_dict = defaultdict(int)
 last_time_u_dict = defaultdict(int)
 last_time2_u_dict = defaultdict(int)
 last_time3_u_dict = defaultdict(int)
 last_time4_u_dict = defaultdict(int)
 last_time5_u_dict = defaultdict(int)
 last_time6_u_dict = defaultdict(int)
 tfl_std_u_dict = defaultdict(float)
 tfl_hist_u_dict = defaultdict(float)
 tfl_hist2_u_dict = defaultdict(float)
 acc_sum_c_dict = defaultdict(int)
 exp_sum_b_dict = defaultdict(int)
 time_sum_b_dict = defaultdict(int)
 acc_std_c_dict = defaultdict(float)
 exp_std_b_dict = defaultdict(float)
 time_std_b_dict = defaultdict(float)
 count_c_dict = defaultdict(int)
 count_b_dict = defaultdict(int)
 acc_sum_up_dict = defaultdict(lambda:defaultdict(int))
acc_hist_up_dict = defaultdict(lambda:defaultdict(float))
acc_hist2_up_dict = defaultdict(lambda:defaultdict(float))
count_up_dict = defaultdict(lambda:defaultdict(int))
count_lecture_up_dict = defaultdict(lambda:defaultdict(int))
last_time_up_dict = defaultdict(lambda:defaultdict(int))
last_time2_up_dict = defaultdict(lambda:defaultdict(int))
tfl_hist_up_dict = defaultdict(lambda:defaultdict(int))
tfl_hist2_up_dict = defaultdict(lambda:defaultdict(int))
exp_sum_up_dict = defaultdict(lambda:defaultdict(int))
exp_hist_up_dict = defaultdict(lambda:defaultdict(float))
exp_hist2_up_dict = defaultdict(lambda:defaultdict(float))
time_sum_up_dict = defaultdict(lambda:defaultdict(int))
time_hist_up_dict = defaultdict(lambda:defaultdict(float))
time_hist2_up_dict = defaultdict(lambda:defaultdict(float))
count_upp_dict = defaultdict(lambda:defaultdict(int))
acc_sum_uc_dict = defaultdict(lambda:defaultdict(int))
acc_hist_uc_dict = defaultdict(lambda:defaultdict(float))
acc_hist2_uc_dict = defaultdict(lambda:defaultdict(float))
count_uc_dict = defaultdict(lambda:defaultdict(int))
count_lecture_uc_dict = defaultdict(lambda:defaultdict(int))
last_time_uc_dict = defaultdict(lambda:defaultdict(int))
last_time2_uc_dict = defaultdict(lambda:defaultdict(int))
tfl_hist_uc_dict = defaultdict(lambda:defaultdict(float))
tfl_hist2_uc_dict = defaultdict(lambda:defaultdict(float))
exp_sum_ub_dict = defaultdict(lambda:defaultdict(int))
exp_hist_ub_dict = defaultdict(lambda:defaultdict(float))
exp_hist2_ub_dict = defaultdict(lambda:defaultdict(float))
time_sum_ub_dict = defaultdict(lambda:defaultdict(int))
time_hist_ub_dict = defaultdict(lambda:defaultdict(float))
time_hist2_ub_dict = defaultdict(lambda:defaultdict(float))
count_ub_dict = defaultdict(lambda:defaultdict(int))
train = add_feats(train, acc_sum_u_dict, acc_std_u_dict, acc_hist_u_dict, acc_hist2_u_dict,
exp_sum_u_dict, exp_std_u_dict, exp_hist_u_dict, exp_hist2_u_dict,
time_sum_u_dict, time_std_u_dict, time_hist_u_dict, time_hist2_u_dict,
dpt_sum_u_dict, dpt_std_u_dict, dpt_hist_u_dict, dpt_hist2_u_dict,
dif_sum_u_dict, dif_std_u_dict, dif_hist_u_dict, dif_hist2_u_dict,
count_u_dict, count_lecture_u_dict,
last_time_u_dict, last_time2_u_dict, last_time3_u_dict,
last_time4_u_dict, last_time5_u_dict, last_time6_u_dict,
tfl_std_u_dict, tfl_hist_u_dict, tfl_hist2_u_dict,
acc_sum_c_dict, exp_sum_b_dict, time_sum_b_dict,
acc_std_c_dict, exp_std_b_dict, time_std_b_dict,
count_c_dict, count_b_dict,
acc_sum_up_dict, acc_hist_up_dict, acc_hist2_up_dict,
count_up_dict, count_lecture_up_dict, last_time_up_dict, last_time2_up_dict,
tfl_hist_up_dict,tfl_hist2_up_dict,
exp_sum_up_dict, exp_hist_up_dict, exp_hist2_up_dict,
time_sum_up_dict, time_hist_up_dict, time_hist2_up_dict,
count_upp_dict,
acc_sum_uc_dict, acc_hist_uc_dict, acc_hist2_uc_dict,
count_uc_dict, count_lecture_uc_dict, last_time_uc_dict, last_time2_uc_dict,
tfl_hist_uc_dict, tfl_hist2_uc_dict,
exp_sum_ub_dict, exp_hist_ub_dict, exp_hist2_ub_dict,
time_sum_ub_dict, time_hist_ub_dict, time_hist2_ub_dict,
count_ub_dict)
 valid = add_feats(valid, acc_sum_u_dict, acc_std_u_dict, acc_hist_u_dict, acc_hist2_u_dict,
exp_sum_u_dict, exp_std_u_dict, exp_hist_u_dict, exp_hist2_u_dict,
time_sum_u_dict, time_std_u_dict, time_hist_u_dict, time_hist2_u_dict,
dpt_sum_u_dict, dpt_std_u_dict, dpt_hist_u_dict, dpt_hist2_u_dict,
dif_sum_u_dict, dif_std_u_dict, dif_hist_u_dict, dif_hist2_u_dict,
count_u_dict, count_lecture_u_dict,
last_time_u_dict, last_time2_u_dict, last_time3_u_dict,
last_time4_u_dict, last_time5_u_dict, last_time6_u_dict,
tfl_std_u_dict, tfl_hist_u_dict, tfl_hist2_u_dict,
acc_sum_c_dict, exp_sum_b_dict, time_sum_b_dict,
acc_std_c_dict, exp_std_b_dict, time_std_b_dict,
count_c_dict, count_b_dict,
acc_sum_up_dict, acc_hist_up_dict, acc_hist2_up_dict,
count_up_dict, count_lecture_up_dict, last_time_up_dict, last_time2_up_dict,
tfl_hist_up_dict,tfl_hist2_up_dict,
exp_sum_up_dict, exp_hist_up_dict, exp_hist2_up_dict,
time_sum_up_dict, time_hist_up_dict, time_hist2_up_dict,
count_upp_dict,
acc_sum_uc_dict, acc_hist_uc_dict, acc_hist2_uc_dict,
count_uc_dict, count_lecture_uc_dict, last_time_uc_dict, last_time2_uc_dict,
tfl_hist_uc_dict, tfl_hist2_uc_dict,
exp_sum_ub_dict, exp_hist_ub_dict, exp_hist2_ub_dict,
time_sum_ub_dict, time_hist_ub_dict, time_hist2_ub_dict,
count_ub_dict )<drop_column>
result = train_targets_scored.drop(columns=target_cols)\ 
.merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = result[target_cols].values
score = 0
for i in range(y_true.shape[1]):
_score = log_loss(y_true[:,i], y_pred[:,i])
 score += _score / y_true.shape[1]
logger.info(f"Final result: {score}" )
<define_variables><EOS> 
sub = submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False)
 sub.head()