Kaggle competition: Recruit Restaurant Visitor Forecasting
<SOS> metric: RMSLE Kaggle data source: recruit-restaurant-visitor-forecasting<concatenate>
import numpy as np
import pandas as pd
from sklearn import ensemble, neighbors, linear_model, metrics, preprocessing
from datetime import datetime
import glob, re
embedding_matrix = np.concatenate([build_matrix(tokenizer.word_index, f)for f in EMBEDDING_PATHS], axis=-1)
 str_='embedding matrix done'
os.system('echo '+str_ )<set_options>
data = {
'ar': pd.read_csv('.. /input/air_reserve.csv'),
'as': pd.read_csv('.. /input/air_store_info.csv'),
'tra': pd.read_csv('.. /input/air_visit_data.csv'),
'hol': pd.read_csv('.. /input/date_info.csv' ).rename(columns={'calendar_date': 'visit_date'}),
'hr': pd.read_csv('.. /input/hpg_reserve.csv'),
'hs': pd.read_csv('.. /input/hpg_store_info.csv'),
'tes': pd.read_csv('.. /input/sample_submission.csv'),
'id': pd.read_csv('.. /input/store_id_relation.csv')
 }
del train_df, tokenizer
gc.collect()<define_variables>
data['hr'] = data['hr'].merge(data['id'], how='inner', on=['hpg_store_id'])
 data['hr'].sample(5 )
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
EPOCHS=4<set_options>
for df in ['ar', 'hr']:
data[df]['visit_datetime'] = pd.to_datetime(data[df]['visit_datetime'])
 data[df]['visit_datetime'] = data[df]['visit_datetime'].dt.date
data[df]['reserve_datetime'] = pd.to_datetime(data[df]['reserve_datetime'])
 data[df]['reserve_datetime'] = data[df]['reserve_datetime'].dt.date
data[df]['reserve_datetime_diff'] = data[df].apply(
 lambda r:(r['visit_datetime'] - r['reserve_datetime'] ).days, axis=1)
 data[df] = data[df].groupby(['air_store_id', 'visit_datetime'], as_index=False)[['reserve_datetime_diff',
'reserve_visitors']].sum().rename(columns={'visit_datetime':'visit_date'})
 data['hr'].sample(5 )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def build_model_1(embedding_matrix, num_aux_targets):
words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 y = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 y = add([y, Dense(128*4, activation='relu' )(y)])
 y = add([y, Dense(128*4, activation='relu' )(y)])
 z = Attention(MAX_LEN )(x)
 z = add([z, Dense(128*2, activation='relu' )(y)])
 z = add([z, Dense(128*2, activation='relu' )(y)])
 result = concatenate([y, z])
 result = Dense(1, activation='sigmoid' )(y)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(y)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<train_model>
lbl = preprocessing.LabelEncoder()
data['as']['air_genre_name'] = lbl.fit_transform(data['as']['air_genre_name'])
 data['as']['air_area_name'] = lbl.fit_transform(data['as']['air_area_name'])
 data['as'].sample(3 )
str_='training starting'
os.system('echo '+str_)
 checkpoint_predictions = []
weights = []
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for epoch in range(EPOCHS):
str_='starting epoch: ' + str(epoch)
 os.system('echo '+str_)
 model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
sample_weight=[sample_weights, np.ones_like(sample_weights)],
callbacks = [
LearningRateScheduler(lambda _: 1e-3*(0.55**epoch))
] 
)
 preds = model.predict(x_test, batch_size=2048)
 checkpoint_predictions.append(preds[0].flatten())
 weights.append(2*epoch)
 str_='finished epoch: ' + str(epoch)
 os.system('echo '+str_ )<normalization>
data['tra']['visit_date'] = pd.to_datetime(data['tra']['visit_date'])
 data['tra']['day_of_week'] = data['tra']['visit_date'].dt.dayofweek
data['tra']['month'] = data['tra']['visit_date'].dt.month
data['tra']['year'] = data['tra']['visit_date'].dt.year
data['tra']['visit_date'] = data['tra']['visit_date'].dt.date
predictions_lstm = np.average(checkpoint_predictions, weights=weights, axis=0 )<save_model>
data['hol'] = data['hol'].drop(columns=['day_of_week'])
 data['hol']['visit_date'] = pd.to_datetime(data['hol']['visit_date'])
 data['hol']['visit_date'] = data['hol']['visit_date'].dt.date
data['hol'].sample(4 )
model.save('my_model.h5' )<set_options>
data['tes']['visit_date'] = data['tes']['id'].map(lambda x: str(x ).split('_')[2])
 data['tes']['air_store_id'] = data['tes']['id'].map(lambda x: '_'.join(x.split('_')[:2]))
data['tes']['visit_date'] = pd.to_datetime(data['tes']['visit_date'])
 data['tes']['day_of_week'] = data['tes']['visit_date'].dt.dayofweek
data['tes']['month'] = data['tes']['visit_date'].dt.month
data['tes']['year'] = data['tes']['visit_date'].dt.year
data['tes']['visit_date'] = data['tes']['visit_date'].dt.date
data['tes'].sample(4 )
del model
gc.collect()<compute_train_metric>
train = pd.merge(data['tra'], data['hol'], how='left', on=['visit_date'])
 train = pd.merge(data['tra'], stores, how='left', on=['air_store_id', 'day_of_week'] )
def ensemble_predictions(predictions, weights, type_="linear"):
assert np.isclose(np.sum(weights), 1.0)
 if type_ == "linear":
res = np.average(predictions, weights=weights, axis=0)
 elif type_ == "harmonic":
res = np.average([1 / p for p in predictions], weights=weights, axis=0)
 return 1 / res
elif type_ == "geometric":
numerator = np.average(
 [np.log(p)for p in predictions], weights=weights, axis=0 
)
 res = np.exp(numerator / sum(weights))
return res
elif type_ == "rank":
res = np.average([rankdata(p)for p in predictions], weights=weights, axis=0)
 return res /(len(res)+ 1)
 return res<save_to_csv>
test = pd.merge(data['tes'], data['hol'], how='left', on=['visit_date'])
 test = pd.merge(data['tes'], stores, how='left', on=['air_store_id', 'day_of_week'])
 test.sample(3 )
submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False )<define_variables>
data['ar'] = data['ar'].merge(data['hr'], how='left', on=['air_store_id', 'visit_date', 'reserve_datetime_diff', 'reserve_visitors'])
 data['ar'].sample(4 )
debug = False<define_variables>
train = train.merge(data['ar'], how='left', on=['air_store_id', 'visit_date'])
 test = test.merge(data['ar'], how='left', on=['air_store_id', 'visit_date'] )
start_time = time.time()
package_dir = ".. /input/pytorchpretrainedberthaqishen/pytorch-pretrained-bert/pytorch-pretrained-BERT/"
sys.path = [package_dir] + sys.path<import_modules>
col = [c for c in train if c not in ['id', 'air_store_id', 'visit_date', 'visitors']]
train = train.fillna(-1)
 test = test.fillna(-1)
 train.sample(4 )
TensorDataset)
 Ttokenizer = TreebankWordTokenizer()
symbols_to_isolate = '.,?!-;*"…:—() %
symbols_to_delete = '
🍕\r🐵😑\xa0\ue014\t\uf818\uf04a\xad😢🐶️\uf0e0😜😎👊\u200b\u200e😁عدويهصقأناخلىبمغر😍💖💵Е👎😀😂\u202a\u202c🔥😄🏻💥ᴍʏʀᴇɴᴅᴏᴀᴋʜᴜʟᴛᴄᴘʙғᴊᴡɢ😋👏שלוםבי😱‼\x81エンジ故障\u2009🚌ᴵ͞🌟😊😳😧🙀😐😕\u200f👍😮😃😘אעכח💩💯⛽🚄🏼ஜ😖ᴠ🚲‐😟😈💪🙏🎯🌹😇💔😡\x7f👌ἐὶήιὲκἀίῃἴξ🙄Ｈ😠\ufeff\u2028😉😤⛺🙂\u3000تحكسة👮💙فزط😏🍾🎉😞\u2008🏾😅😭👻😥😔😓🏽🎆🍻🍽🎶🌺🤔😪\x08‑🐰🐇🐱🙆😨🙃💕𝘊𝘦𝘳𝘢𝘵𝘰𝘤𝘺𝘴𝘪𝘧𝘮𝘣💗💚地獄谷улкнПоАН🐾🐕😆ה🔗🚽歌舞伎🙈😴🏿🤗🇺🇸мυтѕ⤵🏆🎃😩\u200a🌠🐟💫💰💎эпрд\x95🖐🙅⛲🍰🤐👆🙌\u2002💛🙁👀🙊🙉\u2004ˢᵒʳʸᴼᴷᴺʷᵗʰᵉᵘ\x13🚬🤓\ue602😵άοόςέὸתמדףנרךצט😒͝🆕👅👥👄🔄🔤👉👤👶👲🔛🎓\uf0b7\uf04c\x9f\x10成都😣⏺😌🤑🌏😯ех😲Ἰᾶὁ💞🚓🔔📚🏀👐\u202d💤🍇\ue613小土豆🏡❔⁉\u202f👠》कर्मा🇹🇼🌸蔡英文🌞🎲レクサス😛外国人关系Сб💋💀🎄💜🤢َِьыгя不是\x9c\x9d🗑\u2005💃📣👿༼つ༽😰ḷЗз▱ц￼🤣卖温哥华议会下降你失去所有的钱加拿大坏税骗子🐝ツ🎅\x85🍺آإشء🎵🌎͟ἔ油别克🤡🤥😬🤧й\u2003🚀🤴ʲшчИОРФДЯМюж😝🖑ὐύύ特殊作戦群щ💨圆明园קℐ🏈😺🌍⏏ệ🍔🐮🍁🍆🍑🌮🌯🤦\u200d𝓒𝓲𝓿𝓵안영하세요ЖљКћ🍀😫🤤ῦ我出生在了可以说普通话汉语好极🎼🕺🍸🥂🗽🎇🎊🆘🤠👩🖒🚪天一家⚲\u2006⚭⚆⬭⬯⏖新✀╌🇫🇷🇩🇪🇮🇬🇧😷🇨🇦ХШ🌐\x1f杀鸡给猴看ʁ𝗪𝗵𝗲𝗻𝘆𝗼𝘂𝗿𝗮𝗹𝗶𝘇𝗯𝘁𝗰𝘀𝘅𝗽𝘄𝗱📺ϖ\u2000үսᴦᎥһͺ\u2007հ\u2001ɩｙｅ൦ｌƽｈ𝐓𝐡𝐞𝐫𝐮𝐝𝐚𝐃𝐜𝐩𝐭𝐢𝐨𝐧Ƅᴨןᑯ໐ΤᏧ௦Іᴑ܁𝐬𝐰𝐲𝐛𝐦𝐯𝐑𝐙𝐣𝐇𝐂𝐘𝟎ԜТᗞ౦〔Ꭻ𝐳𝐔𝐱𝟔𝟓𝐅🐋ﬃ💘💓ё𝘥𝘯𝘶💐🌋🌄🌅𝙬𝙖𝙨𝙤𝙣𝙡𝙮𝙘𝙠𝙚𝙙𝙜𝙧𝙥𝙩𝙪𝙗𝙞𝙝𝙛👺🐷ℋ𝐀𝐥𝐪🚶𝙢Ἱ🤘ͦ💸ج패티Ｗ𝙇ᵻ👂👃ɜ🎫\uf0a7БУі🚢🚂ગુજરાતીῆ🏃𝓬𝓻𝓴𝓮𝓽𝓼☘﴾̯﴿₽\ue807𝑻𝒆𝒍𝒕𝒉𝒓𝒖𝒂𝒏𝒅𝒔𝒎𝒗𝒊👽😙\u200cЛ‒🎾👹⎌🏒⛸公寓养宠物吗🏄🐀🚑🤷操美𝒑𝒚𝒐𝑴🤙🐒欢迎来到阿拉斯ספ𝙫🐈𝒌𝙊𝙭𝙆𝙋𝙍𝘼𝙅ﷻ🦄巨收赢得白鬼愤怒要买额ẽ🚗🐳𝟏𝐟𝟖𝟑𝟕𝒄𝟗𝐠𝙄𝙃👇锟斤拷𝗢𝟳𝟱𝟬⦁マルハニチロ株式社⛷한국어ㄸㅓ니͜ʖ𝘿𝙔₵𝒩ℯ𝒾𝓁𝒶𝓉𝓇𝓊𝓃𝓈𝓅ℴ𝒻𝒽𝓀𝓌𝒸𝓎𝙏ζ𝙟𝘃𝗺𝟮𝟭𝟯𝟲👋🦊多伦🐽🎻🎹⛓🏹🍷🦆为和中友谊祝贺与其想象对法如直接问用自己猜本传教士没积唯认识基督徒曾经让相信耶稣复活死怪他但当们聊些政治题时候战胜因圣把全堂结婚孩恐惧且栗谓这样还♾🎸🤕🤒⛑🎁批判检讨🏝🦁🙋😶쥐스탱트뤼도석유가격인상이경제황을렵게만들지않록잘관리해야합다캐나에서대마초와화약금의품런성분갈때는반드시허된사용🔫👁凸ὰ💲🗯𝙈Ἄ𝒇𝒈𝒘𝒃𝑬𝑶𝕾𝖙𝖗𝖆𝖎𝖌𝖍𝖕𝖊𝖔𝖑𝖉𝖓𝖐𝖜𝖞𝖚𝖇𝕿𝖘𝖄𝖛𝖒𝖋𝖂𝕴𝖟𝖈𝕸👑🚿💡知彼百\uf005𝙀𝒛𝑲𝑳𝑾𝒋𝟒😦𝙒𝘾𝘽🏐𝘩𝘨ὼṑ𝑱𝑹𝑫𝑵𝑪🇰🇵👾ᓇᒧᔭᐃᐧᐦᑳᐨᓃᓂᑲᐸᑭᑎᓀᐣ🐄🎈🔨🐎🤞🐸💟🎰🌝🛳点击查版🍭𝑥𝑦𝑧ＮＧ👣\uf020っ🏉ф💭🎥Ξ🐴👨🤳🦍\x0b🍩𝑯𝒒😗𝟐🏂👳🍗🕉🐲چی𝑮𝗕𝗴🍒ꜥⲣⲏ🐑⏰鉄リ事件ї💊「」\uf203\uf09a\uf222\ue608\uf202\uf099\uf469\ue607\uf410\ue600燻製シ虚偽屁理屈Г𝑩𝑰𝒀𝑺🌤𝗳𝗜𝗙𝗦𝗧🍊ὺἈἡχῖΛ⤏🇳𝒙ψՁմեռայինրւդձ冬至ὀ𝒁🔹🤚🍎𝑷🐂💅𝘬𝘱𝘸𝘷𝘐𝘭𝘓𝘖𝘹𝘲𝘫کΒώ💢ΜΟΝΑΕ🇱♲𝝈↴💒⊘Ȼ🚴🖕🖤🥘📍👈➕🚫🎨🌑🐻𝐎𝐍𝐊𝑭🤖🎎😼🕷ｇｒｎｔｉｄｕｆｂｋ𝟰🇴🇭🇻🇲𝗞𝗭𝗘𝗤👼📉🍟🍦🌈🔭《🐊🐍\uf10aლڡ🐦\U0001f92f\U0001f92a🐡💳ἱ🙇𝗸𝗟𝗠𝗷🥜さようなら🔼'
CONTRACTION_MAPPING = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
isolate_dict = {ord(c):f' {c} ' for c in symbols_to_isolate}
remove_dict = {ord(c):f'' for c in symbols_to_delete}
warnings.filterwarnings(action='once')
 device = torch.device('cuda')
 print('Import done! Time past %.2f secs' %(time.time() - start_time))<categorify>
def RMSLE(y, pred):
return metrics.mean_squared_error(y, pred)**0.5
def _apply_df(args):
df, func, kwargs = args
return df.apply(func, **kwargs)
 def apply_by_multiprocessing(df, func, **kwargs):
workers = kwargs.pop('workers')
 pool = multiprocessing.Pool(processes=workers)
 result = pool.map(_apply_df, [(d, func, kwargs)
 for d in np.array_split(df, workers)])
 pool.close()
return pd.concat(list(result))<choose_model_class>
etc = ensemble.ExtraTreesRegressor(n_estimators=225, max_depth=5, n_jobs=-1, random_state=3)
 knn = neighbors.KNeighborsRegressor(n_jobs=-1, n_neighbors=4)
 etc.fit(train[col], np.log1p(train['visitors'].values))
knn.fit(train[col], np.log1p(train['visitors'].values))
class BertForJigsaw(BertPreTrainedModel):
def __init__(self, config, out_dim=7):
super(BertForJigsaw, self ).__init__(config)
 self.bert = BertModel(config)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.linear = nn.Linear(config.hidden_size, out_dim)
 self.apply(self.init_bert_weights)
 def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None):
_, pooled_output = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False)
 pooled_output = self.dropout(pooled_output)
 logits = self.linear(pooled_output)
 return logits
class GPT2ClassificationHeadModel(GPT2PreTrainedModel):
def __init__(self, config, clf_dropout=0.4, out_dim=8):
super(GPT2ClassificationHeadModel, self ).__init__(config)
 self.transformer = GPT2Model(config)
 self.linear = nn.Linear(config.n_embd * 2, out_dim)
 nn.init.normal_(self.linear.weight, std = 0.02)
 nn.init.normal_(self.linear.bias, 0)
 self.apply(self.init_weights)
 def set_num_special_tokens(self, num_special_tokens):
pass
def forward(self, input_ids, position_ids=None, token_type_ids=None, lm_labels=None, past=None, **kwargs):
hidden_states, presents = self.transformer(input_ids, position_ids, token_type_ids, past)
 if isinstance(hidden_states, list):
hidden_states = hidden_states[-1]
avg_pool = torch.mean(hidden_states, 1)
 max_pool, _ = torch.max(hidden_states, 1)
 h_conc = torch.cat(( avg_pool, max_pool), 1)
 return self.linear(h_conc)
 class XLNetForJigSaw(XLNetPreTrainedModel):
def __init__(self, config, out_dim):
super(XLNetForJigSaw, self ).__init__(config)
 self.attn_type = config.attn_type
self.same_length = config.same_length
self.summary_type = "last"
self.transformer = XLNetModel(config, output_attentions=False, keep_multihead_output=False)
 self.dense = nn.Linear(config.d_model, config.d_model)
 self.activation = nn.Tanh()
self.linear = nn.Linear(config.d_model, out_dim, bias=True)
 self.apply(self.init_xlnet_weights)
 def forward(self, input_ids, seg_id=None, input_mask=None,
mems=None, perm_mask=None, target_mapping=None, inp_q=None,
target=None, output_all_encoded_layers=True, head_mask=None, **kargs):
output, hidden_states, new_mems = self.transformer(input_ids, seg_id, input_mask,
mems, perm_mask, target_mapping, inp_q,
output_all_encoded_layers, head_mask)
 first_token_tensor = output[:, 0]
pooled_output = self.dense(first_token_tensor)
 pooled_output = self.activation(pooled_output)
 return self.linear(pooled_output )<categorify>
test['visitors'] =(etc.predict(test[col])/ 2)+(knn.predict(test[col])/ 2)
 test['visitors'] = np.expm1(test['visitors'] ).clip(lower=0.)
sub1 = test[['id', 'visitors']].copy()
def convert_line(row, max_seq_length, tokenizer, model_name='bert'):
guid = row['id']
text_a = row['comment_text']
if 'label' in row.keys() :
label = row['label']
else:
label = None
tokens_a = tokenizer.tokenize(text_a)
 if 'bert' in model_name:
if len(tokens_a)> max_seq_length - 2:
tokens_a = tokens_a[:(max_seq_length - 2)]
tokens = ["[CLS]"] + tokens_a + ["[SEP]"]
input_ids = tokenizer.convert_tokens_to_ids(tokens)
 else:
if len(tokens_a)> max_seq_length:
tokens_a = tokens_a[:max_seq_length]
tokens = tokens_a
input_ids = tokenizer.convert_tokens_to_ids(tokens_a)
 segment_ids = [0] * len(tokens)
 input_mask = [1] * len(input_ids)
 padding = [0] *(max_seq_length - len(input_ids))
input_ids += padding
input_mask += padding
segment_ids += padding
return input_ids, input_mask, segment_ids, label<categorify>
dfs = {re.search('/([^/\.]*)\.csv', fn ).group(1):
pd.read_csv(fn)for fn in glob.glob('.. /input/*.csv')}
for k, v in dfs.items() : locals() [k] = v
def preprocess(x):
for k, v in CONTRACTION_MAPPING.items() :
x = x.replace(' %s ' % k, ' %s ' % v)
 x = x.translate(remove_dict)
 x = x.translate(isolate_dict)
 x = Ttokenizer.tokenize(x)
 x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x<data_type_conversions>
date_info['weight'] =(( date_info.index + 1)/ len(date_info)) ** 5
def get_input_data(test_data):
all_input_ids, all_input_mask, all_segment_ids, all_label_ids = [], [], [], []
for i,(input_ids, input_mask, segment_ids, label)in test_data.items() :
all_input_ids.append(input_ids)
 all_input_mask.append(input_mask)
 all_segment_ids.append(segment_ids)
 all_label_ids.append(label)
 all_input_ids = torch.tensor(all_input_ids, dtype=torch.long)
 all_input_mask = torch.tensor(all_input_mask, dtype=torch.long)
 all_segment_ids = torch.tensor(all_segment_ids, dtype=torch.long)
 try:
all_label_ids = torch.tensor(all_label_ids, dtype=torch.float32)
 except:
pass
return all_input_ids, all_input_mask, all_segment_ids, all_label_ids
print('Def functions done! Time past %.2f secs' %(time.time() - start_time))<load_from_csv>
visit_data = air_visit_data.merge(date_info, left_on = 'visit_date', right_on = 'calendar_date', how = 'left')
 visit_data.drop('calendar_date', axis=1, inplace = True)
 visit_data['visitors'] = visit_data.visitors.map(pd.np.log1p)
 visit_data.head()
print('Loading data...')
 df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 if debug:
df = df.loc[:25]
test_ids = df['id'].tolist()
df['comment_text'] = df['comment_text'].astype(str)
 print('Preprocessing...')
 df['comment_text'] = apply_by_multiprocessing(df['comment_text'], preprocess, workers=4)
 print('Done! Time past %.2f secs' %(time.time() - start_time))<randomize_order>
wmean = lambda x:(( x.weight * x.visitors ).sum() / x.weight.sum())
 visitors = visit_data.groupby(['air_store_id', 'day_of_week', 'holiday_flg'] ).apply(wmean ).rename('visitors' ).reset_index()
visitors.head()
try:
model_dir = '.. /input/jigsawmodels/bert_small_v2_29bin_naus_300seq/bert_small_v2_29bin_naus_300seq/'
max_seq_length = 300
short_length = 100
tokenizer = BertTokenizer.from_pretrained(model_dir, do_lower_case=True)
 print('Converting data to sequences...')
 test_data = apply_by_multiprocessing(df, convert_line, axis=1, max_seq_length=max_seq_length, tokenizer=tokenizer, model_name='bert', workers=4)
 all_input_ids, all_input_mask, all_segment_ids, all_label_ids = get_input_data(test_data)
 long_idx =(all_input_ids[:, short_length-max_seq_length:].sum(1)> 0 ).nonzero().squeeze().numpy()
short_idx =(all_input_ids[:, short_length-max_seq_length:].sum(1)== 0 ).nonzero().squeeze().numpy()
print('Done! Time past %.2f secs' %(time.time() - start_time))
print('Loading model from %s...' % model_dir)
 bert_config = BertConfig(os.path.join(model_dir, 'config.json'))
model = BertForJigsaw(bert_config, out_dim=29+6)
 model.load_state_dict(torch.load('%s/pytorch_model.bin' % model_dir))
model.to(device)
 model.eval()
print('Done! Time past %.2f secs' %(time.time() - start_time))
print('Predicting model Bert Small...')
 predictions_bert_small = np.zeros(df.shape[0])
 with torch.no_grad() :
for i, idx in enumerate([short_idx, long_idx]):
test_data = TensorDataset(all_input_ids[idx], all_input_mask[idx], all_segment_ids[idx])if i == 1 else \
TensorDataset(all_input_ids[idx, :short_length], all_input_mask[idx, :short_length], all_segment_ids[idx, :short_length])
 test_sampler = SequentialSampler(test_data)
 test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=128)
 pred = []
for input_ids, input_mask, segment_ids in tqdm(test_dataloader):
input_ids = input_ids.to(device)
 input_mask = input_mask.to(device)
 segment_ids = segment_ids.to(device)
 logits = model(input_ids, segment_ids, input_mask, labels=None)
 logits = torch.sigmoid(logits ).detach().cpu().numpy()
pred.append(logits[:, :-6])
 predictions_bert_small[idx] = np.vstack(pred ).mean(1)
 print('Done! Time past %.2f secs' %(time.time() - start_time))
except:
print('Something wrong with Bert Small.')
 traceback.print_exc()<string_transform>
sample_submission['air_store_id'] = sample_submission.id.map(lambda x: '_'.join(x.split('_')[:-1]))
sample_submission['calendar_date'] = sample_submission.id.map(lambda x: x.split('_')[2])
 sample_submission.drop('visitors', axis=1, inplace=True)
 sample_submission = sample_submission.merge(date_info, on='calendar_date', how='left')
 sample_submission = sample_submission.merge(visitors, on=[
'air_store_id', 'day_of_week', 'holiday_flg'], how='left' )
try:
model_dir = '.. /input/jigsawmodels/bert_large_v2_99bin_250seq/bert_large_v2_99bin_250seq/'
max_seq_length = 250
short_length = 100
tokenizer = BertTokenizer.from_pretrained(model_dir, do_lower_case=True)
 print('Converting data to sequences...')
 test_data = apply_by_multiprocessing(df, convert_line, axis=1, max_seq_length=max_seq_length, tokenizer=tokenizer, model_name='bert', workers=4)
 all_input_ids, all_input_mask, all_segment_ids, all_label_ids = get_input_data(test_data)
 long_idx =(all_input_ids[:, short_length-max_seq_length:].sum(1)> 0 ).nonzero().squeeze().numpy()
short_idx =(all_input_ids[:, short_length-max_seq_length:].sum(1)== 0 ).nonzero().squeeze().numpy()
print('Done! Time past %.2f secs' %(time.time() - start_time))
print('Loading model from %s...' % model_dir)
 bert_config = BertConfig(os.path.join(model_dir, 'config.json'))
model = BertForJigsaw(bert_config, out_dim=99)
 model.load_state_dict(torch.load('%s/pytorch_model.bin' % model_dir))
model.to(device)
 model.eval()
print('Done! Time past %.2f secs' %(time.time() - start_time))
print('Predicting model Bert Large')
 predictions_bert_large = np.zeros(df.shape[0])
 with torch.no_grad() :
for i, idx in enumerate([short_idx, long_idx]):
test_data = TensorDataset(all_input_ids[idx], all_input_mask[idx], all_segment_ids[idx])if i == 1 else \
TensorDataset(all_input_ids[idx, :short_length], all_input_mask[idx, :short_length], all_segment_ids[idx, :short_length])
 test_sampler = SequentialSampler(test_data)
 test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=64)
 pred = []
for input_ids, input_mask, segment_ids in tqdm(test_dataloader):
input_ids = input_ids.to(device)
 input_mask = input_mask.to(device)
 segment_ids = segment_ids.to(device)
 logits = model(input_ids, segment_ids, input_mask, labels=None)
 logits = torch.sigmoid(logits ).detach().cpu().numpy()
pred.append(logits)
 predictions_bert_large[idx] = np.vstack(pred ).mean(1)
 print('Done! Time past %.2f secs' %(time.time() - start_time))
except:
print('Something wrong with Bert Large.')
 traceback.print_exc()<categorify>
sample_submission.visitors.isnull().sum()
try:
model_dir = '.. /input/jigsawmodels/xlnet_large_9bin_220seq/xlnet_large_9bin_220seq/'
max_seq_length = 220
tokenizer = XLNetTokenizer.from_pretrained(model_dir)
 print('Converting to sequences...')
 test_data = []
for i, row in tqdm(df.iterrows()):
test_data.append(convert_line(
 row,
max_seq_length=max_seq_length,
tokenizer=tokenizer,
model_name='xlnet' 
))
test_data = pd.Series(test_data)
 all_input_ids, all_input_mask, all_segment_ids, all_label_ids = get_input_data(test_data)
 print('Done! Time past %.2f secs' %(time.time() - start_time))
test_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids)
 test_sampler = SequentialSampler(test_data)
 test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=32)
 print('Loading model from %s...' % model_dir)
 xlnet_config = XLNetConfig(os.path.join(model_dir, 'config.json'))
model = XLNetForJigSaw(xlnet_config, out_dim=9)
 model.load_state_dict(torch.load('%s/pytorch_model.bin' % model_dir))
model.to(device)
 model.eval()
print('Done! Time past %.2f secs' %(time.time() - start_time))
print('Predicting model xlnet_large_v2_9bin_220seq')
 predictions_xlnet = []
with torch.no_grad() :
for input_ids, input_mask, segment_ids in tqdm(test_dataloader):
input_ids = input_ids.to(device)
 input_mask = input_mask.to(device)
 segment_ids = segment_ids.to(device)
 logits = model(input_ids)
 logits = torch.sigmoid(logits ).detach().cpu().numpy()
predictions_xlnet.append(logits)
 predictions_xlnet = np.vstack(predictions_xlnet ).mean(1)
 print('Done! Time past %.2f secs' %(time.time() - start_time))
except:
print('Something wrong with xlnet_large_v2_9bin_220seq.')
 traceback.print_exc()<randomize_order>
missings = sample_submission.visitors.isnull()
sample_submission.loc[missings, 'visitors'] = sample_submission[missings].merge(
 visitors[visitors.holiday_flg == 0], on=('air_store_id', 'day_of_week'),
how='left')['visitors_y'].values
try:
model_dir = '.. /input/jigsawmodels/gpt2_29bin_350seq_aus/gpt2_29bin_350seq_aus/'
max_seq_length = 350
tokenizer = GPT2Tokenizer.from_pretrained(model_dir)
 print('Converting data to sequences...')
 test_data = apply_by_multiprocessing(df, convert_line, axis=1, max_seq_length=max_seq_length, tokenizer=tokenizer, model_name='gpt2', workers=4)
 all_input_ids, all_input_mask, all_segment_ids, all_label_ids = get_input_data(test_data)
 print('Done! Time past %.2f secs' %(time.time() - start_time))
test_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids)
 test_sampler = SequentialSampler(test_data)
 test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=64)
 print('Loading model from %s...' % model_dir)
 gpt2_config = GPT2Config(os.path.join(model_dir, 'config.json'))
model = GPT2ClassificationHeadModel(gpt2_config, out_dim=29+6)
 model.load_state_dict(torch.load('%s/pytorch_model.bin' % model_dir))
model.to(device)
 model.eval()
print('Done! Time past %.2f secs' %(time.time() - start_time))
print('Predicting model GPT2...')
 predictions_gpt2 = []
with torch.no_grad() :
for input_ids, input_mask, segment_ids in tqdm(test_dataloader):
if(time.time() - start_time)> 7140:
print('STOP GPT2 FOR THE TIME SACK.')
 break
input_ids = input_ids.to(device)
 input_mask = input_mask.to(device)
 segment_ids = segment_ids.to(device)
 logits = model(
 input_ids=input_ids,
token_type_ids=segment_ids,
attention_mask=input_mask, 
)
 logits = torch.sigmoid(logits ).detach().cpu().numpy()
predictions_gpt2.append(logits[:, :-6])
 predictions_gpt2 = np.vstack(predictions_gpt2 ).mean(1)
 print('Done! Time past %.2f secs' %(time.time() - start_time))
except:
print('Something wrong with GPT2:')
 traceback.print_exc()
print('But the program is still running' )<define_variables>
sample_submission.isnull().sum()
predictions = np.zeros(df.shape[0])
 try:
predictions += predictions_bert_small * 0.9
except: pass
try:
predictions += predictions_bert_large * 1.0
except: pass
try:
predictions += predictions_gpt2 * 0.7
except: pass
try:
predictions += predictions_xlnet * 0.8
except: pass<save_to_csv>
missings = sample_submission.visitors.isnull()
sample_submission.loc[missings, 'visitors'] = sample_submission[missings].merge(
 visitors[['air_store_id', 'visitors']].groupby('air_store_id' ).mean().reset_index() ,
on='air_store_id', how='left')['visitors_y'].values
df_prediction = pd.DataFrame({
'id': test_ids,
'prediction': predictions,
})
 df_prediction.to_csv('./submission.csv', index=False)
 print('Output done! Time past %.2f secs' %(time.time() - start_time))<import_modules>
sample_submission.isnull().sum()
class SequenceBucketCollator() :
def __init__(self, choose_length, sequence_index, length_index, label_index=None):
self.choose_length = choose_length
self.sequence_index = sequence_index
self.length_index = length_index
self.label_index = label_index
def __call__(self, batch):
batch = [torch.stack(x)for x in list(zip(*batch)) ]
sequences = batch[self.sequence_index]
lengths = batch[self.length_index]
length = self.choose_length(lengths)
 mask = torch.arange(start=maxlen, end=0, step=-1)< length
padded_sequences = sequences[:, mask]
batch[self.sequence_index] = padded_sequences
if self.label_index is not None:
return [x for i, x in enumerate(batch)if i != self.label_index], batch[self.label_index]
return batch
CUSTOM_TABLE = str.maketrans(
 {
"\xad": None,
"\x7f": None,
"\ufeff": None,
"\u200b": None,
"\u200e": None,
"\u202a": None,
"\u202c": None,
"‘": "'",
"’": "'",
"`": "'",
"“": '"',
"”": '"',
"«": '"',
"»": '"',
"ɢ": "G",
"ɪ": "I",
"ɴ": "N",
"ʀ": "R",
"ʏ": "Y",
"ʙ": "B",
"ʜ": "H",
"ʟ": "L",
"ғ": "F",
"ᴀ": "A",
"ᴄ": "C",
"ᴅ": "D",
"ᴇ": "E",
"ᴊ": "J",
"ᴋ": "K",
"ᴍ": "M",
"Μ": "M",
"ᴏ": "O",
"ᴘ": "P",
"ᴛ": "T",
"ᴜ": "U",
"ᴡ": "W",
"ᴠ": "V",
"ĸ": "K",
"в": "B",
"м": "M",
"н": "H",
"т": "T",
"ѕ": "S",
"—": "-",
"–": "-",
} 
)
 WORDS_REPLACER = [ 
("sh*t", "shit"), 
("s**t", "shit"), 
("f*ck", "fuck"), 
("fu*k", "fuck"), 
("f**k", "fuck"), 
("f*****g", "fucking"), 
("f***ing", "fucking"), 
("f**king", "fucking"), 
("p*ssy", "pussy"), 
("p***y", "pussy"), 
("pu**y", "pussy"), 
("p*ss", "piss"), 
("b*tch", "bitch"), 
("bit*h", "bitch"), 
("h*ll", "hell"), 
("h**l", "hell"), 
("cr*p", "crap"), 
("d*mn", "damn"), 
("stu*pid", "stupid"), 
("st*pid", "stupid"), 
("n*gger", "nigger"), 
("n***ga", "nigger"), 
("f*ggot", "faggot"), 
("scr*w", "screw"), 
("pr*ck", "prick"), 
("g*d", "god"), 
("s*x", "sex"), 
("a*s", "ass"), 
("a**hole", "asshole"), 
("a***ole", "asshole"), 
("a**", "ass"),
]
REGEX_REPLACER = [ 
(re.compile(pat.replace("*", "\*"), flags=re.IGNORECASE), repl)
 for pat, repl in WORDS_REPLACER
]
RE_SPACE = re.compile(r"\s")
 RE_MULTI_SPACE = re.compile(r"\s+")
 NMS_TABLE = dict.fromkeys(
 i for i in range(sys.maxunicode + 1)if unicodedata.category(chr(i)) == "Mn" 
)
 HEBREW_TABLE = {i: "א" for i in range(0x0590, 0x05FF)}
ARABIC_TABLE = {i: "ا" for i in range(0x0600, 0x06FF)}
CHINESE_TABLE = {i: "是" for i in range(0x4E00, 0x9FFF)}
KANJI_TABLE = {i: "ッ" for i in range(0x2E80, 0x2FD5)}
HIRAGANA_TABLE = {i: "ッ" for i in range(0x3041, 0x3096)}
KATAKANA_TABLE = {i: "ッ" for i in range(0x30A0, 0x30FF)}
TABLE = dict()
TABLE.update(CUSTOM_TABLE)
 TABLE.update(NMS_TABLE)
 TABLE.update(CHINESE_TABLE)
 TABLE.update(HEBREW_TABLE)
 TABLE.update(ARABIC_TABLE)
 TABLE.update(HIRAGANA_TABLE)
 TABLE.update(KATAKANA_TABLE)
 TABLE.update(KANJI_TABLE)
 EMOJI_REGEXP = emoji.get_emoji_regexp()
UNICODE_EMOJI_MY = {
k: f" EMJ {v.strip(':' ).replace('_', ' ')} "
for k, v in emoji.UNICODE_EMOJI_ALIAS.items()
}
def my_demojize(string: str)-> str:
def replace(match):
return UNICODE_EMOJI_MY.get(match.group(0), match.group(0))
return re.sub("\ufe0f", "", EMOJI_REGEXP.sub(replace, string))
def normalize(text: str)-> str:
text = my_demojize(text)
 text = RE_SPACE.sub(" ", text)
 text = unicodedata.normalize("NFKD", text)
 text = text.translate(TABLE)
 text = RE_MULTI_SPACE.sub(" ", text ).strip()
for pattern, repl in REGEX_REPLACER:
text = pattern.sub(repl, text)
 return text
PORTER_STEMMER = PorterStemmer()
LANCASTER_STEMMER = LancasterStemmer()
SNOWBALL_STEMMER = SnowballStemmer("english")
 def word_forms(word):
yield word
yield word.lower()
yield word.upper()
yield word.capitalize()
yield PORTER_STEMMER.stem(word)
 yield LANCASTER_STEMMER.stem(word)
 yield SNOWBALL_STEMMER.stem(word)
 def maybe_get_embedding(word, model):
for form in word_forms(word):
if form in model:
return model[form]
word = word.strip("-'")
 for form in word_forms(word):
if form in model:
return model[form]
return None
def gensim_to_embedding_matrix(word2index, path):
model = KeyedVectors.load(path, mmap="r")
 embedding_matrix = np.zeros(( max(word2index.values())+ 1, model.vector_size), dtype=np.float32)
 unknown_words = []
for word, i in word2index.items() :
maybe_embedding = maybe_get_embedding(word, model)
 if maybe_embedding is not None:
embedding_matrix[i] = maybe_embedding
else:
unknown_words.append(word)
 return embedding_matrix, unknown_words<load_from_csv>
sample_submission['visitors'] = sample_submission.visitors.map(pd.np.expm1)
 sub2 = sample_submission[['id', 'visitors']].copy()
sub_merge = pd.merge(sub1, sub2, on='id', how='inner' )
%%time
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<categorify>
sub_merge['visitors'] =(sub_merge['visitors_x'] * sub_merge['visitors_y'])**(1/2)
 sub_merge[['id', 'visitors']].to_csv('sub_geo_mean.csv', index = False)
 sub_merge[['id', 'visitors']].head()
<feature_engineering><EOS> 
sub_merge['visitors'] = 2/(1/sub_merge['visitors_x'] + 1/sub_merge['visitors_y'])
 sub_merge[['id', 'visitors']].to_csv('sub_hrm_mean.csv', index = False)
 sub_merge[['id', 'visitors']].head()