Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<import_modules>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
from catboost import CatBoostClassifier, cv, Pool
from sklearn.model_selection import train_test_split<split>
import numpy as np
import random
import pandas as pd
import joblib
xtrain,xtest,ytrain,ytest = train_test_split(X,Y,train_size=0.82,random_state=42 )<choose_model_class>
_ = np.seterr(divide='ignore', invalid='ignore' )
clf =CatBoostClassifier(eval_metric='Accuracy',use_best_model=True,random_seed=42 )<train_model>
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
clf.fit(xtrain,ytrain,cat_features=cate_features_index,eval_set=(xtest,ytest), early_stopping_rounds=50 )<count_missing_values>
print('start read train data...')
 train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
test_id = data_tittest.PassengerId
data_tittest.isnull().sum()<predict_on_test>
print('start handle lecture data...' )
prediction = clf.predict(data_tittest )<save_to_csv>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
df_sub = pd.DataFrame()
df_sub['PassengerId'] = test_id
df_sub['Survived'] = prediction.astype(np.int)
 df_sub.to_csv('gender_submission.csv', index=False )<load_from_csv>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission_df = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv' )<filter>
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
train_df[(train_df['target'] == 0)&(train_df['keyword'].isnull())]<filter>
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()
train_df[(train_df['target'] == 1)&(train_df['keyword'].isnull())]<count_unique_values>
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )
print(f"No.of unique keywords in class 0: {train_df[(train_df['target'] == 0)]['keyword'].value_counts().shape[0]}")
 print(f"No.of unique keywords in class 1: {train_df[(train_df['target'] == 1)]['keyword'].value_counts().shape[0]}")
 print(f"Total no.of unique keywords: {train_df['keyword'].value_counts().shape[0]}" )<define_variables>
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )
URL_PATTERN = re.compile(r"(https:\/\/\S+)|(http:\/\/\S+)|(www\.\S+)")
 HTML_TAGS_PATTERN = re.compile(r'<.*>')
 ALPHA_NUMERIC_PATTERN = re.compile(r"\w*[:,-]*(\d+[:,-]*)+\d*\w*")
 PUNCTUATION_PATTERN = re.compile(r'[^a-zA-Z ]')
 MENTIONS_PATTERN = re.compile(r'@[\w]*')
 HASH_TAGS_PATTERN = re.compile(r'
UNWANTED_WORDS_PATTERN = re.compile(r'&amp;|RT: \S+:|RT \S+:|FYI|CAD|RT |GMT|UTC|JST|\s[b-zB-Z]\s|ST|nsfw')
 STOPWORDS = set(stopwords.words('english'))
def get_punctuations(text):
return PUNCTUATION_PATTERN.findall(text)
 punctuations = train_df.apply(lambda x: get_punctuations(x['text']), axis=1)
 punctuations = punctuations[punctuations.notnull() ].explode().value_counts()
fig = px.bar(x=punctuations.index, y=punctuations.values)
 fig.update_layout(title_text='Punctuation count', xaxis_title='Punctuations', yaxis_title='Count' )<categorify>
del(train_lectures)
 gc.collect()
def replace_contractions(tweet):
tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 return tweet<categorify>
user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'])
 user_lecture_agg=user_lecture_agg.astype('int16' )
class TweetTransformer(BaseEstimator, TransformerMixin):
def __init__(self, remove_stopwords=False):

 self.remove_stopwords_ = remove_stopwords
def replace_contractions(self, tweet):

 return replace_contractions(tweet)
 def replace_urls(self, tweet):

 return URL_PATTERN.sub('', tweet)
 def replace_unwanted_words(self, tweet):

 return UNWANTED_WORDS_PATTERN.sub(' ', tweet)
 def replace_mentions(self, tweet):

 return MENTIONS_PATTERN.sub('', tweet)
 def replace_alpha_nums(self, tweet):

 return ALPHA_NUMERIC_PATTERN.sub('', tweet)
 def remove_emoji_characters(self, tweet):

 return tweet.encode('ascii', 'ignore' ).decode('ascii')
 def replace_punctuations(self, tweet):

 return PUNCTUATION_PATTERN.sub(' ', tweet)
 def modify_empty_and_rare_utterances(self, tweet):

 if tweet.strip() == '':
tweet = 'no text'
tweet = tweet.lower()
tweet = re.sub('lo+l', 'laughing out loud', tweet)
 tweet = re.sub('coo+l', 'cool', tweet)
 tweet = re.sub('(calif|cal)', 'california', tweet)
 tweet = tweet.replace('rd', 'road')
 tweet = tweet.replace('nyc', 'new york city')
 tweet = tweet.replace('sismo', 'earthquake')
 tweet = tweet.replace('detactado', 'detected')
 tweet = re.sub(' +', ' ', tweet)
 tweet = re.sub('^ +', '', tweet)
 return tweet
def remove_stopwords(self, tweet):

 tokens = word_tokenize(tweet)
 for i, token in enumerate(tokens):
if token in STOPWORDS:
del tokens[i]
return " ".join(tokens)
 def transform_helper(self, tweet):

 tweet = self.replace_contractions(tweet)
 tweet = self.replace_urls(tweet)
 tweet = self.replace_unwanted_words(tweet)
 tweet = self.replace_mentions(tweet)
 tweet = self.replace_alpha_nums(tweet)
 tweet = self.remove_emoji_characters(tweet)
 tweet = self.replace_punctuations(tweet)
 tweet = self.modify_empty_and_rare_utterances(tweet)
 if self.remove_stopwords_:
tweet = self.remove_stopwords(tweet)
 return tweet
def fit(self, X=None):
return self
def transform(self, X):

 return X.map(lambda x: self.transform_helper(x))<train_model>
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['user_interaction_count'] = cum['cumcount']
train_df['user_interaction_timestamp_mean'] = train_df['timestamp']/cum['cumcount']
train_df['user_lecture_sum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_sum=train_df.user_lecture_sum.astype('int16')
 train_df.user_interaction_count=train_df.user_interaction_count.astype('int16')
 train_df['user_interaction_timestamp_mean']=train_df['user_interaction_timestamp_mean']/(1000*3600)
 train_df.user_interaction_timestamp_mean=train_df.user_interaction_timestamp_mean.astype('float32')
class NNLMEmbeddingsTransformer(BaseEstimator, TransformerMixin):
def fit(self, X=None):
return self
def transform(self, X):
return nnlm_embed(X.values )<train_model>
del cum
gc.collect()
class USEEmbeddingsTransformer(BaseEstimator, TransformerMixin):
def fit(self, X=None):
return self
def transform(self, X):
return use_embed(X.values )<feature_engineering>
print('start handle train_df...' )
use_embeddings_pipeline = Pipeline([ 
('tweet_transformer', TweetTransformer()), 
('use_embeddings_transformer', USEEmbeddingsTransformer())
 ])
 tfidf_pipeline = Pipeline([ 
('tweet_transformer', TweetTransformer(remove_stopwords=True)) , 
('tfidf_vectorizer', TfidfVectorizer())
 ])
 data_prep_pipeline = FeatureUnion([ 
('use_embeddings', use_embeddings_pipeline), 
('tfidf', tfidf_pipeline)
 ])
 X_train = data_prep_pipeline.fit_transform(train_df['text'])
 y_train = train_df['target']
X_test = data_prep_pipeline.transform(test_df['text'] )<set_options>
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True )
h2o.init()<categorify>
content_explation_agg=train_df[["content_id","prior_question_had_explanation",target]].groupby(["content_id","prior_question_had_explanation"])[target].agg(['mean'] )
train_h2o = H2OFrame(X_train.todense())
 x = train_h2o.columns
y = 'target'
train_h2o[y] = H2OFrame(y_train.values ).asfactor()
X_test_h2o = H2OFrame(X_test.todense() )<train_model>
content_explation_agg=content_explation_agg.unstack()
content_explation_agg=content_explation_agg.reset_index()
content_explation_agg.columns = ['content_id', 'content_explation_false_mean','content_explation_true_mean']
aml = H2OAutoML(max_models=20, max_runtime_secs=18000, seed=1)
 aml.train(x=x, y=y, training_frame=train_h2o )<predict_on_test>
content_explation_agg['explation_impact'] =(( content_explation_agg['content_explation_true_mean'] -
content_explation_agg['content_explation_false_mean'])/
content_explation_agg['content_explation_false_mean'])
 content_explation_agg['explation_impact'] = np.where(content_explation_agg['explation_impact'] < 0, 0, content_explation_agg['explation_impact'])
 content_explation_agg['explation_impact'] = np.where(content_explation_agg['content_explation_false_mean']==0, content_explation_agg['content_explation_true_mean'], content_explation_agg['explation_impact'] )
predictions = aml.predict(X_test_h2o )<save_to_csv>
content_explation_agg.content_id=content_explation_agg.content_id.astype('int16')
 content_explation_agg.content_explation_false_mean=content_explation_agg.content_explation_false_mean.astype('float16')
 content_explation_agg.content_explation_true_mean=content_explation_agg.content_explation_true_mean.astype('float16')
 content_explation_agg.explation_impact = content_explation_agg.explation_impact.astype('float16' )
submission_df['target'] = predictions['predict'].as_data_frame().values
submission_df.to_csv('submissions.csv', index=False)
 FileLink('submissions.csv' )<set_options>
print('start handle attempt_no...' )
%matplotlib inline<load_from_csv>
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'] ).astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()<load_from_csv>
attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]
test_data = pd.read_csv("/kaggle/input/titanic/test.csv" )<categorify>
print('start handle timestamp...')
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )
<categorify>
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
max_timestamp_u.user_id=max_timestamp_u.user_id.astype('int32' )
def age_fill(columns):
age = columns[0]
pclass = columns[1]
if pd.isnull(age):
if pclass == 1:
return int(train_data[train_data['Pclass']==1]['Age'].mean())
 elif pclass == 2:
return int(train_data[train_data['Pclass']==2]['Age'].mean())
 else:
return int(train_data[train_data['Pclass']==3]['Age'].mean())
 else:
return age<feature_engineering>
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
max_timestamp_u2 = train_df[['user_id','lagtime']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u2.columns = ['user_id', 'max_time_stamp2']
max_timestamp_u2.user_id=max_timestamp_u2.user_id.astype('int32' )
train_data['Age'] = train_data[['Age','Pclass']].apply(age_fill, axis=1 )<drop_column>
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
lagtime_mean=train_df['lagtime'].mean()
train_df['lagtime'].fillna(lagtime_mean, inplace=True )
<drop_column>
train_df['lagtime']=train_df['lagtime']/(1000*3600)
 train_df.lagtime=train_df.lagtime.astype('float32' )
train_data.drop('Cabin', axis=1, inplace=True )<correct_missing_values>
train_df['lagtime2'] = train_df.groupby('user_id')['timestamp'].shift(2)
 max_timestamp_u3 = train_df[['user_id','lagtime2']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u3.columns = ['user_id', 'max_time_stamp3']
max_timestamp_u3.user_id=max_timestamp_u3.user_id.astype('int32')
 train_df['lagtime2']=train_df['timestamp']-train_df['lagtime2']
lagtime_mean2=train_df['lagtime2'].mean()
train_df['lagtime2'].fillna(lagtime_mean2, inplace=True )
train_data.dropna(inplace=True )<categorify>
train_df['lagtime2']=train_df['lagtime2']/(1000*3600)
 train_df.lagtime2=train_df.lagtime2.astype('float32' )
sex = pd.get_dummies(train_data['Sex'],drop_first=True)
 embark = pd.get_dummies(train_data['Embarked'],drop_first=True )<drop_column>
train_df['lagtime3'] = train_df.groupby('user_id')['timestamp'].shift(3)
 train_df['lagtime3']=train_df['timestamp']-train_df['lagtime3']
lagtime_mean3=train_df['lagtime3'].mean()
train_df['lagtime3'].fillna(lagtime_mean3, inplace=True)
 train_df['lagtime3']=train_df['lagtime3']/(1000*3600)
 train_df.lagtime3=train_df.lagtime3.astype('float32' )
train_data.drop(['Sex','Embarked', 'Name', 'Ticket'], axis=1, inplace=True )<concatenate>
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('float16' )
train_data = pd.concat([train_data, sex, embark], axis=1 )<feature_engineering>
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
test_data['Age'] = test_data[['Age','Pclass']].apply(age_fill, axis=1 )<categorify>
delta_prior_question_elapsed_time_mean=train_df['delta_prior_question_elapsed_time'].mean()
train_df['delta_prior_question_elapsed_time'].fillna(delta_prior_question_elapsed_time_mean, inplace=True)
 train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )
def fill_fare(col):
pclass = col[0]
fare = col[1]
if pd.isnull(fare):
if pclass == 1:
return int(train_data[train_data['Pclass']==1]['Fare'].mean())
 elif pclass == 2:
return int(train_data[train_data['Pclass']==2]['Fare'].mean())
 else:
return int(train_data[train_data['Pclass']==3]['Fare'].mean())
 else:
return fare<feature_engineering>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 user_agg = train_df.groupby('user_id')['lag'].agg(['sum', 'count'] ).astype('int16')
 cum['cumsum'].fillna(0, inplace=True)
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_count'] = cum['cumsum']
train_df['user_uncorrect_count'] = cum['cumcount']-cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correctness'].fillna(0.67, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_count=train_df.user_correct_count.astype('int16')
 train_df.user_uncorrect_count=train_df.user_uncorrect_count.astype('int16' )
test_data['Fare'] = test_data[['Pclass','Fare']].apply(fill_fare, axis=1 )<drop_column>
del cum
gc.collect()
test_data.drop('Cabin', axis=1, inplace=True )<categorify>
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16' )
sex = pd.get_dummies(test_data['Sex'],drop_first=True)
 embark = pd.get_dummies(test_data['Embarked'],drop_first=True )<drop_column>
cum = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_true_count'] = cum['cumsum']
train_df['explanation_false_count'] = cum['cumcount']-cum['cumsum']
train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_true_count=train_df.explanation_true_count.astype('int16')
 train_df.explanation_false_count=train_df.explanation_false_count.astype('int16' )
test_data.drop(['Sex','Embarked', 'Name', 'Ticket'], axis=1, inplace=True )<concatenate>
del cum
gc.collect()
test_data = pd.concat([test_data, sex, embark], axis=1 )<save_to_csv>
content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )
y = train_data["Survived"]
features = ['Pclass','Age','SibSp','Parch','Fare','male','Q','S']
X = train_data[features]
X_test = test_data[features]
model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=1)
 model.fit(X, y)
 predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<train_model>
train_df['task_container_uncor_count'] = train_df['task_container_id'].map(task_container_agg['count']-task_container_agg['sum'] ).astype('int32')
 train_df['task_container_cor_count'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )
LogModel = LogisticRegression()
LogModel.fit(X,y )<predict_on_test>
content_elapsed_time_agg=train_df.groupby('content_id')['prior_question_elapsed_time'].agg(['mean'])
 content_had_explanation_agg=train_df.groupby('content_id')['prior_question_had_explanation'].agg(['mean'] )
Logpred = LogModel.predict(X_test )<save_to_csv>
print('start questions data...' )
output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': Logpred})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<import_modules>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'}
)
import numpy as np
import pandas as pd
import os
from keras import models, layers, Sequential
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier<load_from_csv>
bundle_agg = questions_df.groupby('bundle_id')['question_id'].agg(['count'] )
train_df = pd.read_csv('.. /input/train.csv')
 train_df.head()<load_from_csv>
questions_df['content_sub_bundle'] = questions_df['bundle_id'].map(bundle_agg['count'] ).astype('int8' )
test_df = pd.read_csv('.. /input/test.csv')
 test_df.head()<concatenate>
questions_df['tags'].fillna('188', inplace=True )
merged_df = pd.concat(( train_df.drop(['Survived'], axis = 1), test_df))
merged_df.head()<import_modules>
def gettags(tags,num):
tags_splits=tags.split(" ")
 result=''
for t in tags_splits:
x=int(t)
 if(x<32*(num+1)and x>=32*num):
result=result+' '+t
return result
import seaborn as sb
from matplotlib import pyplot as plt<groupby>
for num in range(0,6):
questions_df["tags"+str(num)] = questions_df["tags"].apply(lambda row: gettags(row,num))
le = LabelEncoder()
le.fit(np.unique(questions_df['tags'+str(num)].values))
questions_df['tags'+str(num)]=questions_df[['tags'+str(num)]].apply(le.transform )
train_df.groupby(['Pclass'])['Survived'].sum() / train_df.groupby(['Pclass'])['Survived'].count()<groupby>
questions_df_dict = {
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
}
questions_df = questions_df.astype(questions_df_dict )
train_df.groupby(['Sex'])['Survived'].sum() / train_df.groupby(['Sex'])['Survived'].count()<groupby>
questions_df.drop(columns=['tags'], inplace=True )
train_df.groupby(['SibSp'])['Survived'].sum() / train_df.groupby(['SibSp'])['Survived'].count()<groupby>
questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32' )
train_df.groupby(['Parch'])['Survived'].sum() / train_df.groupby(['Parch'])['Survived'].count()<feature_engineering>
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )
train_df['Family'] = train_df['SibSp'] + train_df['Parch']<groupby>
questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True )
train_df.groupby(['Family'])['Survived'].sum() / train_df.groupby(['Family'])['Survived'].count()<drop_column>
del content_explation_agg
train_df.drop(['Family'], axis = 1, inplace = True )<sort_values>
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')
 questions_df['content_uncorrect_count'] = questions_df['content_id'].map(content_agg['count']-content_agg['sum'] ).astype('int32')
 questions_df['content_correct_count'] = questions_df['content_id'].map(content_agg['sum'] ).astype('int32' )
ticket_num_records = train_df.groupby(['Ticket'] ).size().sort_values(ascending=False ).to_dict()
train_df.groupby(['Ticket'] ).size().sort_values(ascending=False ).head()<feature_engineering>
questions_df['content_elapsed_time_mean'] = questions_df['content_id'].map(content_elapsed_time_agg['mean'])
 questions_df.content_elapsed_time_mean=questions_df.content_elapsed_time_mean.astype('float16')
 questions_df['content_had_explanation_mean'] = questions_df['content_id'].map(content_had_explanation_agg['mean'])
 questions_df.content_had_explanation_mean=questions_df.content_had_explanation_mean.astype('float16' )
train_df['Companion'] = train_df['Ticket'].apply(lambda x: ticket_num_records[x] )<groupby>
del content_elapsed_time_agg
del content_had_explanation_agg
gc.collect()
train_df.groupby(['Companion'])['Survived'].sum() / train_df.groupby(['Companion'])['Survived'].count()<drop_column>
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )
train_df.drop(['Companion'], axis = 1, inplace = True )<groupby>
part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32' )
train_df.groupby(pd.cut(train_df["Fare"], np.arange(0, 350, 25)))['Survived'].sum() / train_df.groupby(pd.cut(train_df["Fare"], np.arange(0, 350, 25)))['Survived'].count()<groupby>
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16' )
train_df['CabinId'] = train_df['Cabin'].apply(lambda x: 'None' if pd.isna(x)else x[0])
 train_df.groupby(['CabinId'])['Survived'].sum() / train_df.groupby(['CabinId'])['Survived'].count()<drop_column>
del content_agg
del bundle_agg
del part_agg
gc.collect()
train_df.drop(['CabinId'], axis = 1, inplace = True )<groupby>
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'explation_impact':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16'
}
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags0',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'part_bundle_id',
'content_sub_bundle',
'prior_question_had_explanation'
]
features=list(features_dict.keys())
train_df.groupby(['Embarked'])['Survived'].sum() / train_df.groupby(['Embarked'])['Survived'].count()<feature_engineering>
flag_lgbm=True
clfs = list()
train_df['Title'] = train_df['Name'].apply(lambda x: re.compile('.+?[,][\s] (.*?)[\.][\s].+' ).findall(x)[0])
 train_df.groupby(['Title'])['Survived'].sum() / train_df.groupby(['Title'])['Survived'].count()<groupby>
params = {
'num_leaves': 355,
'max_bin': 256+128+64,
'feature_fraction': 0.548,
'bagging_fraction': 0.648,
'bagging_freq': 11,
'min_data_in_leaf': 31+64+8,
'max_depth': 15+4,
'objective': 'binary',
'learning_rate': 0.0212345,
"boosting_type": "gbdt",
"metric": 'auc',
'seed': 123
}
train_df.groupby(['Title'])['Survived'].count()<drop_column>
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1200*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08, random_state=42)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df = train_df_clf.sample(frac=0.03, random_state=42)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df))
train_df.drop(['Title'], axis = 1, inplace = True )<filter>
del train_df_clf
del valid_df
gc.collect()
np.nanmean(train_df[train_df['Name'].str.contains('Master')]['Age'] )<filter>
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=7000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 fig,ax = plt.subplots(figsize=(15,15))
lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)
 plt.show()
del tr_data
del va_data
gc.collect()
np.nanmean(train_df[train_df['Name'].str.contains('Miss')]['Age'] )<feature_engineering>
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
merged_df['Title'] = merged_df['Name'].apply(lambda x: re.compile('.+?[,][\s] (.*?)[\.][\s].+' ).findall(x)[0] )<feature_engineering>
MAX_SEQ = 240
ACCEPTED_USER_CONTENT_SIZE = 2
EMBED_SIZE = 256
BATCH_SIZE = 64+32
DROPOUT = 0.1
boymean = np.nanmean(train_df[train_df['Name'].str.contains('Master.')]['Age'])
 girlmean = np.nanmean(train_df[train_df['Name'].str.contains('Miss.')]['Age'])
 meanage = np.nanmean(train_df['Age'] )<feature_engineering>
class FFN(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 class FFN0(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN0, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.layer_normal = nn.LayerNorm(state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 x=self.layer_normal(x)
 return self.dropout(x )
merged_df['Age'] = np.where(np.isnan(merged_df['Age'])&(merged_df['Title'] == 'Master'), boymean, merged_df['Age'])
 merged_df['Age'] = np.where(np.isnan(merged_df['Age'])&(merged_df['Title'] == 'Miss'), girlmean, merged_df['Age'])
 merged_df['Age'] = merged_df['Age'].fillna(meanage )<feature_engineering>
def future_mask(seq_length):
future_mask =(np.triu(np.ones([seq_length, seq_length]), k = 1)).astype('bool')
 return torch.from_numpy(future_mask )
merged_df['Fare'] = merged_df['Fare'].fillna(np.nanmedian(merged_df['Fare']))
merged_df['Fare'] = merged_df['Fare'].apply(lambda x: 1 if x > 75.0 else 0 )<feature_engineering>
class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.ffn0 = FFN0(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x1 = self.ffn0(att_output)
 x = self.dropout(self.layer_normal_2(x + x1 + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq = MAX_SEQ, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
merged_df['Cabin'] = merged_df['Cabin'].apply(lambda x: 0 if pd.isna(x)else 1 )<data_type_conversions>
skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 group = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/group.pkl.zip")
 m_path = "/kaggle/input/v4-fork-of-riiid-sakt-model-full/sakt_model.pt"
n_skill = len(skills)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 nn_model = SAKTModel(n_skill, embed_dim = EMBED_SIZE)
 try:
nn_model.load_state_dict(torch.load(m_path))
except:
nn_model.load_state_dict(torch.load(m_path, map_location='cpu'))
nn_model.to(device)
 nn_model.eval()
merged_df['Embarked'] = merged_df['Embarked'].fillna('N' )<feature_engineering>
class TestDataset(Dataset):
def __init__(self, samples, test_df, n_skill, max_seq = MAX_SEQ):
super(TestDataset, self ).__init__()
self.samples, self.user_ids, self.test_df = samples, [x for x in test_df["user_id"].unique() ], test_df
self.n_skill, self.max_seq = n_skill, max_seq
def __len__(self):
return self.test_df.shape[0]
def __getitem__(self, index):
test_info = self.test_df.iloc[index]
user_id = test_info['user_id']
target_id = test_info['content_id']
content_id_seq = np.zeros(self.max_seq, dtype=int)
 answered_correctly_seq = np.zeros(self.max_seq, dtype=int)
 if user_id in self.samples.index:
content_id, answered_correctly = self.samples[user_id]
seq_len = len(content_id)
 if seq_len >= self.max_seq:
content_id_seq = content_id[-self.max_seq:]
answered_correctly_seq = answered_correctly[-self.max_seq:]
else:
content_id_seq[-seq_len:] = content_id
answered_correctly_seq[-seq_len:] = answered_correctly
x = content_id_seq[1:].copy()
x +=(answered_correctly_seq[1:] == 1)* self.n_skill
questions = np.append(content_id_seq[2:], [target_id])
 return x, questions
merged_df['Family'] = merged_df['Parch'] + merged_df['SibSp']<drop_column>
merged_df.drop(['Name', 'Ticket', 'SibSp', 'Parch'], axis = 1, inplace = True )<groupby>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
train_df.groupby(pd.cut(train_df["Age"], np.arange(0, 100, 20)))['Survived'].sum() / train_df.groupby(pd.cut(train_df["Age"], np.arange(0, 100, 20)))['Survived'].count()<feature_engineering>
del user_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()
maxAge = train_df['Age'].max()
minAge = train_df['Age'].min()
merged_df['Age'] =(merged_df['Age'] - minAge)/(maxAge - minAge )<categorify>
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_lecture_agg
gc.collect()
dummiesPclass = pd.get_dummies(merged_df['Pclass'], prefix = 'Pclass')
 merged_df = pd.concat([merged_df, dummiesPclass], axis=1)
 merged_df.head()<categorify>
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
max_timestamp_u_dict2=max_timestamp_u2.set_index('user_id' ).to_dict()
max_timestamp_u_dict3=max_timestamp_u3.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del max_timestamp_u2
del max_timestamp_u3
del user_prior_question_elapsed_time
gc.collect()
dummiesFare = pd.get_dummies(merged_df['Fare'], prefix = 'Fare')
 merged_df = pd.concat([merged_df, dummiesFare], axis=1)
 merged_df.head()<groupby>
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()
merged_df.groupby(['Title'])['PassengerId'].count()<feature_engineering>
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]
merged_df['Title'] = merged_df['Title'].apply(lambda x: 'Miss' if(x in ['Mlle', 'Mme', 'Ms'])else x)
 merged_df['Title'] = merged_df['Title'].apply(lambda x: 'Mrs' if(x in ['Dona', 'Lady'])else x)
 merged_df['Title'] = merged_df['Title'].apply(lambda x: 'Mr' if(x == 'Rev')else x)
 merged_df['Title'] = merged_df['Title'].apply(lambda x: x if(x in ['Master', 'Mr', 'Mrs', 'Miss'])else 'Other' )<groupby>
env = riiideducation.make_env()
merged_df.groupby(['Title'])['PassengerId'].count()<categorify>
iter_test = env.iter_test()
prior_test_df = None
prev_test_df = None
le = LabelEncoder()
merged_df['Title'] = le.fit_transform(merged_df['Title'] )<categorify>
N=[0.4,0.6]
dummiesTitle = pd.get_dummies(merged_df['Title'], prefix = 'Title')
 merged_df = pd.concat([merged_df, dummiesTitle], axis=1)
 merged_df.head()<categorify>
%%time
for(test_df, sample_prediction_df)in iter_test:
test_df1 = test_df.copy()
if(prev_test_df is not None):
prev_test_df['answered_correctly'] = eval(test_df1['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 prev_test_df = test_df1.copy()
test_df1 = test_df1[test_df1.content_type_id == False]
test_dataset = TestDataset(group, test_df1, n_skill, max_seq=MAX_SEQ)
 test_dataloader = DataLoader(test_dataset, batch_size = 51200, shuffle=False)
 outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = nn_model(x, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 user_ids = prior_test_df['user_id'].values
targets = prior_test_df[target].values
for user_id, answered_correctly in zip(user_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
prior_test_df = test_df.copy()
question_len=len(test_df[test_df['content_type_id'] == 0])
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(question_len, dtype=np.int16)
 user_lecture_count = np.zeros(question_len, dtype=np.int16)
 user_sum = np.zeros(question_len, dtype=np.int16)
 user_count = np.zeros(question_len, dtype=np.int16)
 task_container_sum = np.zeros(question_len, dtype=np.int32)
 task_container_count = np.zeros(question_len, dtype=np.int32)
 task_container_std = np.zeros(question_len, dtype=np.float16)
 explanation_sum = np.zeros(question_len, dtype=np.int32)
 explanation_count = np.zeros(question_len, dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(question_len, dtype=np.int32)
 attempt_no_count = np.zeros(question_len, dtype=np.int16)
 lagtime = np.zeros(question_len, dtype=np.float32)
 lagtime2 = np.zeros(question_len, dtype=np.float32)
 lagtime3 = np.zeros(question_len, dtype=np.float32)
 i=0
for j,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(content_type_id==1):
x=1
if(content_type_id==0):
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):
lagtime2[i]=lagtime_mean2
lagtime3[i]=lagtime_mean3
else:
lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]
if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):
lagtime3[i]=lagtime_mean3
else:
lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]
max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]
max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
else:
lagtime[i]=lagtime_mean
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime2[i]=lagtime_mean2
max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})
 lagtime3[i]=lagtime_mean3
max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})
 if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=delta_prior_question_elapsed_time_mean
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 i=i+1
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_sum'] = user_lecture_sum
test_df['user_interaction_count'] = user_lecture_count
test_df['user_interaction_timestamp_mean'] = test_df['timestamp']/user_lecture_count
test_df['user_correctness'] = user_sum / user_count
test_df['user_uncorrect_count'] =user_count-user_sum
test_df['user_correct_count'] =user_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_cor_count'] = task_container_sum
test_df['task_container_uncor_count'] =task_container_count-task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_true_count'] = explanation_sum
test_df['explanation_false_count'] = explanation_count-explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime2"]=lagtime2
test_df["lagtime3"]=lagtime3
test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('float16')
 test_df['lagtime']=test_df['lagtime']/(1000*3600)
 test_df.lagtime=test_df.lagtime.astype('float32')
 test_df['lagtime2']=test_df['lagtime2']/(1000*3600)
 test_df.lagtime2=test_df.lagtime2.astype('float32')
 test_df['lagtime3']=test_df['lagtime3']/(1000*3600)
 test_df.lagtime3=test_df.lagtime3.astype('float32')
 test_df['user_interaction_timestamp_mean']=test_df['user_interaction_timestamp_mean']/(1000*3600)
 test_df.user_interaction_timestamp_mean=test_df.user_interaction_timestamp_mean.astype('float32')
 test_df['user_correctness'].fillna(0.67, inplace=True)
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
o2=sub_preds / len(clfs)
 test_df[target] = 0.5 *np.array(outs)+ 0.5 *np.array(o2)
 env.predict(test_df[['row_id', target]] )