Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<string_transform>
warnings.simplefilter(action='ignore', category=FutureWarning )
lemmatizer = WordNetLemmatizer()
def preprocess(ingredients):
ingredients_text = ' '.join(ingredients)
 ingredients_text = ingredients_text.lower()
ingredients_text = ingredients_text.replace('-', ' ')
 words = []
for word in ingredients_text.split() :
if re.findall('[0-9]', word): continue
if len(word)<= 2: continue
if '’' in word: continue
word = lemmatizer.lemmatize(word)
 if len(word)> 0: words.append(word)
 return ' '.join(words)
 for ingredient, expected in [ 
('Eggs', 'egg'), 
('all-purpose flour', 'all purpose flour'), 
('purée', 'purée'), 
('1% low-fat milk', 'low fat milk'), 
('half & half', 'half half'), 
('safetida(powder)', 'safetida(powder)')
 ]:
actual = preprocess([ingredient])
 assert actual == expected, f'"{expected}" is excpected but got "{actual}"'<feature_engineering>
%%time
train_pickle_file = '/kaggle/input/pickling/train.csv.pandas.pickle'
train_data = pickle.load(open(train_pickle_file, 'rb'))
train_data.info()
train['x'] = train['ingredients'].progress_apply(preprocess)
 test['x'] = test['ingredients'].progress_apply(preprocess)
 train.head()<feature_engineering>
class RunningMean:
def __init__(self, WIN_SIZE=20, n_size = 1):
self.n = 0
self.mean = np.zeros(n_size)
 self.cum_sum = 0
self.past_value = 0
self.WIN_SIZE = WIN_SIZE
self.windows = collections.deque(maxlen=WIN_SIZE+1)
 def clear(self):
self.n = 0
self.windows.clear()
def push(self, x):
x = fillna_npwhere_njit(x, self.past_value)
 self.past_value = x
self.windows.append(x)
 self.cum_sum += x
if self.n < self.WIN_SIZE:
self.n += 1
self.mean = self.cum_sum / float(self.n)
 else:
self.cum_sum -= self.windows.popleft()
self.mean = self.cum_sum / float(self.WIN_SIZE)
 def get_mean(self):
return self.mean if self.n else np.zeros(n_size)
 def __str__(self):
return "Current window values: {}".format(list(self.windows))
def fillna_npwhere_njit(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array
vectorizer = make_pipeline(
 TfidfVectorizer(sublinear_tf=True),
FunctionTransformer(lambda x: x.astype('float16'), validate=False)
)
 x_train = vectorizer.fit_transform(train['x'].values)
 x_train.sort_indices()
x_test = vectorizer.transform(test['x'].values )<categorify>
a = RunningMean(WIN_SIZE=10000)
 for index, row in tqdm(train_data[:100000].iterrows()):
a.push(np.array(row))
a.get_mean()
label_encoder = LabelEncoder()
y_train = label_encoder.fit_transform(train['cuisine'].values)
 dict(zip(label_encoder.classes_, label_encoder.transform(label_encoder.classes_)) )<choose_model_class>
class RunningStats:
def __init__(self, WIN_SIZE=20, n_size = 1):
self.n = 0
self.mean = 0
self.run_var = 0
self.WIN_SIZE = WIN_SIZE
self.past_value = 0
self.windows = collections.deque(maxlen=WIN_SIZE+1)
 def clear(self):
self.n = 0
self.windows.clear()
def push(self, x):
x = fillna_npwhere_njit(x, self.past_value)
 self.past_value = x
self.windows.append(x)
 if self.n < self.WIN_SIZE:
self.n += 1
delta = x - self.mean
self.mean += delta / self.n
self.run_var += delta *(x - self.mean)
 else:
x_removed = self.windows.popleft()
old_m = self.mean
self.mean +=(x - x_removed)/ self.WIN_SIZE
self.run_var +=(x + x_removed - old_m - self.mean)*(x - x_removed)
 def get_mean(self):
return self.mean if self.n else np.zeros(n_size)
 def get_var(self):
return self.run_var /(self.n)if self.n > 1 else np.zeros(n_size)
 def get_std(self):
return math.sqrt(self.get_var())
 def get_all(self):
return list(self.windows)
 def __str__(self):
return "Current window values: {}".format(list(self.windows))
estimator = SVC(
 C=80,
kernel='rbf',
gamma=1.7,
coef0=1,
cache_size=500, 
)
 classifier = OneVsRestClassifier(estimator, n_jobs=-1 )<compute_train_metric>
class RunningEWMean:
def __init__(self, WIN_SIZE=20, n_size = 1, lt_mean = None):
if lt_mean is not None:
self.s = lt_mean
else:
self.s = np.zeros(n_size)
 self.past_value = np.zeros(n_size)
 self.alpha = 2 /(WIN_SIZE + 1)
 def clear(self):
self.s = 0
def push(self, x):
x = fillna_npwhere_njit(x, self.past_value)
 self.past_value = x
self.s = self.alpha * x +(1 - self.alpha)* self.s
def get_mean(self):
return self.s
%%time
scores = cross_validate(classifier, x_train, y_train, cv=3)
 scores['test_score'].mean()<train_model>
a = RunningEWMean(WIN_SIZE=10000)
 for index, row in tqdm(train_data[:100000].iterrows()):
a.push(np.array(row))
a.get_mean()
%%time
classifier.fit(x_train, y_train )<categorify>
a = RunningPDA()
for index, row in tqdm(train_data[:100000].iterrows()):
date=row['date']
a.push(np.array(row),date )
y_pred = label_encoder.inverse_transform(classifier.predict(x_train))
y_true = label_encoder.inverse_transform(y_train)
 print(f'accuracy score on train data: {accuracy_score(y_true, y_pred)}')
 def report2dict(cr):
rows = []
for row in cr.split("
"):
parsed_row = [x for x in row.split(" ")if len(x)> 0]
if len(parsed_row)> 0: rows.append(parsed_row)
 measures = rows[0]
classes = defaultdict(dict)
 for row in rows[1:]:
class_label = row[0]
for j, m in enumerate(measures):
classes[class_label][m.strip() ] = float(row[j + 1].strip())
 return classes
report = classification_report(y_true, y_pred)
 pd.DataFrame(report2dict(report)).T<save_to_csv>
train_data[:200000].groupby('date' ).mean().iloc[30,]
y_pred = label_encoder.inverse_transform(classifier.predict(x_test))
test['cuisine'] = y_pred
test[['id', 'cuisine']].to_csv('submission.csv', index=False)
 test[['id', 'cuisine']].head()<import_modules>
class RunningPTI:
def __init__(self,base_value=0):
self.dictionnary = {}
self.base_value = base_value
self.day = -1
def clear(self):
self.dictionnary = {}
self.base_value = 0
def push(self, x, value, date):
if date>self.day:
self.day = date
self.dictionnary = {}
self.past_value = self.dictionnary.get(value)
 self.dictionnary.update({value:x})
 def get_past_value(self):
if self.past_value is None:
self.past_value = self.base_value
return self.past_value
def get_dict(self):
return self.dictionnary
print(os.listdir(".. /input")) 
<load_from_disk>
def get_weights(d, size):
w = [1.]
for k in range(1, size):
w_ = -w[-1] / k *(d - k + 1)
 w.append(w_)
 w = np.array(w[::-1] ).reshape(-1, 1)
 return w
train = pd.read_json('.. /input/train.json')
 test = pd.read_json('.. /input/test.json')
 train['cuisine'].value_counts() [:5]<count_unique_values>
del train_data
dataset = pd.concat([train,test],sort=True)
 ingredients = pd.Series([item for sublist in list(dataset['ingredients'])for item in sublist])\ 
.value_counts()
print(ingredients.head(6))
unique_ing = np.array(ingredients.index)
 print('
',len(unique_ing),'ingrédients différents en tout' )<categorify>
physical_devices = tf.config.list_physical_devices('GPU')
 try:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
 except:
pass
vect = TfidfVectorizer(tokenizer=lambda x: [i.strip() for i in x.split(',')], lowercase=False)
 vect.fit(dataset['ingredients'].apply(','.join))
Xtf_train = vect.transform(train['ingredients'].apply(','.join)).toarray()
Xtf_train = pd.DataFrame(Xtf_train,columns=vect.get_feature_names())
 Xtf_test = vect.transform(test['ingredients'].apply(','.join)).toarray()
Xtf_test = pd.DataFrame(Xtf_test,columns=vect.get_feature_names())
 print(Xtf_train.shape,Xtf_test.shape)
 lb = LabelEncoder()
target = train['cuisine']
y_train = lb.fit_transform(target)
 print(y_train.shape )<compute_train_metric>
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
svc = LinearSVC()
print(cross_val_score(svc,Xtf_train,y_train,cv=3))<predict_on_test>
class CVTuner(kt.engine.tuner.Tuner):
def run_trial(self, trial, X, y, splits, batch_size=32, epochs=1,callbacks=None):
val_losses = []
for train_indices, test_indices in splits:
X_train, X_test = [x[train_indices] for x in X], [x[test_indices] for x in X]
y_train, y_test = [a[train_indices] for a in y], [a[test_indices] for a in y]
if len(X_train)< 2:
X_train = X_train[0]
X_test = X_test[0]
if len(y_train)< 2:
y_train = y_train[0]
y_test = y_test[0]
model = self.hypermodel.build(trial.hyperparameters)
 hist = model.fit(X_train,y_train,
validation_data=(X_test,y_test),
epochs=epochs,
batch_size=batch_size,
callbacks=callbacks)
 val_losses.append([hist.history[k][-1] for k in hist.history])
 val_losses = np.asarray(val_losses)
 self.oracle.update_trial(trial.trial_id, {k:np.mean(val_losses[:,i])for i,k in enumerate(hist.history.keys())})
 self.save_model(trial.trial_id, model )
svc.fit(Xtf_train,y_train)
 y_predict = svc.predict(Xtf_test)
 y_string = lb.inverse_transform(y_predict )<save_to_csv>
class LiteModel:
@classmethod
def from_file(cls, model_path):
return LiteModel(tf.lite.Interpreter(model_path=model_path))
@classmethod
def from_keras_model(cls, kmodel):
converter = tf.lite.TFLiteConverter.from_keras_model(kmodel)
 tflite_model = converter.convert()
return LiteModel(tf.lite.Interpreter(model_content=tflite_model))
def __init__(self, interpreter):
self.interpreter = interpreter
self.interpreter.allocate_tensors()
input_det = self.interpreter.get_input_details() [0]
output_det = self.interpreter.get_output_details() [0]
self.input_index = input_det["index"]
self.output_index = output_det["index"]
self.input_shape = input_det["shape"]
self.output_shape = output_det["shape"]
self.input_dtype = input_det["dtype"]
self.output_dtype = output_det["dtype"]
def predict(self, inp):
inp = inp.astype(self.input_dtype)
 count = inp.shape[0]
out = np.zeros(( count, self.output_shape[1]), dtype=self.output_dtype)
 for i in range(count):
self.interpreter.set_tensor(self.input_index, inp[i:i+1])
 self.interpreter.invoke()
out[i] = self.interpreter.get_tensor(self.output_index)[0]
return out
def predict_single(self, inp):

 inp = np.array([inp], dtype=self.input_dtype)
 self.interpreter.set_tensor(self.input_index, inp)
 self.interpreter.invoke()
out = self.interpreter.get_tensor(self.output_index)
 return out[0]
my_submission = pd.DataFrame({'id':test['id'],'cuisine':y_string})
 my_submission.to_csv('submission.csv', index=False )<load_pretrained>
TRAINING = False
USE_FINETUNE = True
FOLDS = 5
SEED = 42
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv' )
try:
stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr
reload(sys)
 sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
sys.setdefaultencoding('utf8')
 except:
pass
random_state = 101
RS = 101
warnings.filterwarnings('ignore')
 wnl = WordNetLemmatizer() 
<drop_column>
nb_trade = train.groupby('date')['date'].count()
high_volume_days = [i for i, x in enumerate(np.array(nb_trade > 7000)) if x]
def pre_processing_(recipe):
wnl = WordNetLemmatizer()
recipe = [str.lower(ingredient)for ingredient in recipe]
recipe = [delete_brand_(ingredient)for ingredient in recipe]
recipe = [delete_state_(ingredient)for ingredient in recipe]
recipe = [delete_comma_(ingredient)for ingredient in recipe]
recipe = [original_(ingredient)for ingredient in recipe]
recipe = [delete_space_(ingredient)for ingredient in recipe if len(delete_space_(ingredient)) >2]
return recipe
def delete_brand_(ingredient):
ingredient = re.sub("country crock|i can't believe it's not butter!|bertolli|oreo|hellmann's", '', ingredient)
 ingredient = re.sub("red gold|hidden valley|original ranch|frank's|redhot|lipton", '', ingredient)
 ingredient = re.sub("recipe secrets|eggland's best|hidden valley|best foods|knorr|land o lakes", '', ingredient)
 ingredient = re.sub("sargento|johnsonville|breyers|diamond crystal|taco bell|bacardi", '', ingredient)
 ingredient = re.sub("mccormick|crystal farms|yoplait|mazola|new york style panetini", '', ingredient)
 ingredient = re.sub("ragu|soy vay|tabasco|truvía|crescent recipe creations|spice islands", '', ingredient)
 ingredient = re.sub("wish-bone|honeysuckle white|pasta sides|fiesta sides", '', ingredient)
 ingredient = re.sub("veri veri teriyaki|artisan blends|home originals|greek yogurt|original ranch", '', ingredient)
 ingredient = re.sub("jonshonville", '', ingredient)
 ingredient = re.sub("old el paso|pillsbury|progresso|betty crocker|green giant|hellmanns|hellmannâ€", '', ingredient)
 ingredient = re.sub("oscar mayer deli fresh smoked", '', ingredient)
 return ingredient
def delete_state_(ingredient):
ingredient = re.sub('frozen|chopped|ground|fresh|powdered', '', ingredient)
 ingredient = re.sub('sharp|crushed|grilled|roasted|sliced', '', ingredient)
 ingredient = re.sub('cooked|shredded|cracked|minced|finely', '', ingredient)
 return ingredient
def delete_comma_(ingredient):
ingredient = ingredient.split(',')
 ingredient = ingredient[0]
return ingredient
def original_(ingredient):
ingredient = re.sub('[0-9]', '', ingredient)
 ingredient = ingredient.replace("oz.", '')
 ingredient = re.sub('[&%() ®™/]', '', ingredient)
 ingredient = re.sub('[-.]', '', ingredient)
 ingredient = wnl.lemmatize(ingredient)
 return ingredient
def delete_space_(ingredient):
ingredient = ingredient.strip()
return ingredient<choose_model_class>
train = train.query('date > 85' ).reset_index(drop = True)
 train = train.query('date not in @high_volume_days' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0.00001)&(train['resp_2'] > 0.00001)&(train['resp_3'] > 0.00001)&(train['resp_4'] > 0.00001)&(train['resp'] > 0.00001)).astype('int' )
lemmatizer = WordNetLemmatizer()
replaceC = [(' ', ' '),(' ', ' '),('ú', 'u'),('é', 'e'),('è', 'e'),('î', 'i'),('â', 'a'),('í', 'i'),('â€™', ''),('€', ''),('ç', 'c'), 
('half & half', 'halfandhalf'),('half half', 'halfandhalf'),('all purpose', ''), 
('seasoning mix', 'seasoning'),('style seasoning', 'seasoning'),('salt free', 'saltfree'),('taco seasoning reduced sodium', 'taco seasoning'), 
('less sodium taco seasoning', 'low sodium taco seasoning'),('old el paso', ''),('old bay','oldbay'), 
('jerk rub', 'jerk'),('taco bell home originals', ''),('taco bell', ''),('low sodium', ''), 
('frozen broccoli florets', 'broccoliflorets'),('broccoli florets', 'broccoliflorets'), 
('high gluten bread flour', 'highglutenflour'),('high gluten flour', 'highglutenflour'), 
('chapatti', 'chapati'),('corn flour', 'cornflour'),('not low fat', ''),('low fat', ''), 
('cauliflorets florets','cauliflorets'),('cauliflorets flowerets','cauliflorets'),('cauliflower','cauliflorets'),('cauliflower florets','cauliflorets'),('caulifloretsets', 'cauliflorets'), 
('bread flour','flour'),('self raising flour','selfraisingflour'),('self rising cake flour','selfraisingflour'),('self rising flour','selfraisingflour'),
]
rmC = u'®™,.;!() []'’"-_@
p0 = re.compile(r'\s*(oz|ounc|ounce|pound|lb|inch|inches|kg|to)\s*[^a-z]')
 def preprocess_ing(Y, doReplaces=True, rmSpaces=False):
if len(Y)<2: return []
if rmSpaces: Y = [_.replace(' ',' ' ).replace(' ','_')for _ in Y]
ingredients_text = ' '.join(Y)
 ingredients_text = ingredients_text.lower()
if doReplaces:
for rm in rmC:
try:
ingredients_text = ingredients_text.replace(rm, ' ')
 except Exception as e:
print(e)
 print(rm)
 print(ingredients_text)
 for rc in replaceC: ingredients_text = ingredients_text.replace(rc[0], rc[1])
 for rc in replaceC: ingredients_text = ingredients_text.replace(rc[0], rc[1])
 if rmSpaces:
for rc in replaceC[2:]:
ingredients_text = ingredients_text.replace(rc[0].replace(' ','_'), rc[1].replace(' ','_'))
words = []
if rmSpaces: split_char='_'
else: split_char=' '
for word in ingredients_text.split(split_char):
word = re.sub(p0, split_char, word.strip())
 if re.findall('[0-9]', word): continue
if len(word)<= 2: continue
if '’' in word: continue
word = lemmatizer.lemmatize(word)
 if len(word)> 0: words.append(word)
 return words
def preprocess_data(X, preproMode, doReplaces=True, rmSpaces=False):
_X = []
for x in X:
if preproMode=='KK': x['ingredients'] = preprocess_ing(x['ingredients'], doReplaces=doReplaces, rmSpaces=rmSpaces)
 elif preproMode=='AV': x['ingredients'] = pre_processing_(x['ingredients'])
 _X.append(x)
 return _X
def conv_secs(s):
mins, secs = divmod(s,60)
 if mins>=60: hours, mins = divmod(mins,60)
 else: hours=0
if hours<24: return '%02d:%02d:%02d' %(hours, mins, secs)
 else:
days, hours = divmod(hours,24)
 return '%02d-%02d:%02d:%02d' %(days, hours, mins, secs)
 def do_train_fit(model, model_name, USEOVR):
t0 = time.time()
if USEOVR:
model = OneVsRestClassifier(model, n_jobs=-1)
 model_name = 'OVR_'+model_name
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid)
 y_pred_train = model.predict(X_train)
 need_time = time.time() -t0
print('%s %s Acc_T: %.4f Acc_V: %.4f [%s]' %(tfidf_name, model_name, accuracy_score(y_pred_train, y_train), accuracy_score(y_pred_valid, y_valid), conv_secs(need_time)))
 return model, model_name, y_pred_valid 
<prepare_x_and_y>
EWM_5000 = RunningEWMean(WIN_SIZE = 5000)
 EWM_10000 = RunningEWMean(WIN_SIZE = 10000)
 EWM_20000 = RunningEWMean(WIN_SIZE = 20000)
 train_FE = []
for index, row in tqdm(train[['feature_0','feature_1']].iterrows()):
EWM_5000.push(np.float64(np.array(row)))
 EWM_10000.push(np.float64(np.array(row)))
 EWM_20000.push(np.float64(np.array(row)))
 FE = {
'feature_0_EWM_5000' : EWM_5000.get_mean() [0],
'feature_1_EWM_5000' : EWM_5000.get_mean() [1],
'feature_0_EWM_10000' : EWM_10000.get_mean() [0],
'feature_1_EWM_10000' : EWM_10000.get_mean() [1],
'feature_0_EWM_20000' : EWM_20000.get_mean() [0],
'feature_1_EWM_20000' : EWM_20000.get_mean() [1],
}
train_FE.append(FE)
 train_FE = pd.DataFrame(train_FE )
print("Read Dataset...")
 def read_dataset(path):
try:return json.load(open(path, encoding='utf-8'))
except:return json.load(open(path))
_HOMEFOLDER_ = os.environ['HOME']
DATA_FOLDER = '.. /input/'
train = read_dataset(DATA_FOLDER+'train.json')
 submission = read_dataset(DATA_FOLDER+'test.json')
 target = [doc['cuisine'] for doc in train]
print("Label Encode the Target Variable...")
 lb = LabelEncoder()
y = lb.fit_transform(target)
 class_weights_ = class_weight.compute_class_weight('balanced', np.unique(y), y)
 class_weights = {}
for i in range(len(class_weights_)) : class_weights[i]=100/class_weights_[i]
X = train
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.00010, random_state=random_state)
 print(len(X_train), len(X_valid))
print("Prepare text data...")
 def generate_text(data, preproMode, doReplaces=True, rmSpaces=False):
data = preprocess_data(data, preproMode=preproMode, doReplaces=doReplaces, rmSpaces=rmSpaces)
 text_data = [" ".join(doc['ingredients'])for doc in data]
return text_data
doReplaces = False
rmSpaces = False
preproMode = 'KK'
preproMode = 'AV'
train_text = generate_text(X_train, preproMode=preproMode, doReplaces=doReplaces, rmSpaces=rmSpaces)
 valid_text = generate_text(X_valid, preproMode=preproMode, doReplaces=doReplaces, rmSpaces=rmSpaces)
 submission_text = generate_text(submission, preproMode=preproMode, doReplaces=doReplaces, rmSpaces=rmSpaces)
 print("Generation Done! " )<define_search_space>
train = pd.concat([train,train_FE],axis=1)
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0.000001 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )
model_njobs = -1
model_list = [ 
('SVC_C50_g1p4_d3_c1',
SVC(C = 50, gamma=1.4, kernel='rbf', degree=3, coef0=1, shrinking=True, tol=0.0001, probability=True,
cache_size=2000, class_weight=None, verbose=False, max_iter=-1, decision_function_shape=None, random_state=RS),
{'C':[30, 50, 100, 150, 220], 'gamma': [0.1, 0.75, 1, 3, 5], 'degree': [1,2,3,4],
'coef0': [0, 0.5, 1], 'kernel':('linear', 'rbf', 'poly')},
{}), 
('LR_C7_l2_NewCG_max200_dF',
LogisticRegression(penalty='l2', C=8,
max_iter = 200, dual = False,
solver='newton-cg',
multi_class='ovr',
n_jobs = model_njobs,
random_state=RS),
{'C': [0.1,0.5,1,5,10,15], 'solver': ['sag', 'newton-cg', 'lbfgs'],
'max_iter':[100,200,300], 'dual':[True, False], 'multi_class': ['multinomial', 'ovr']},
{}), 
('RFC_150_None',
RandomForestClassifier(n_estimators=150, criterion='gini', max_depth=None, min_samples_split=2,
max_features='log2', max_leaf_nodes=None, min_impurity_decrease=0.0,
min_impurity_split=None, bootstrap=True, oob_score=False, n_jobs=model_njobs,
random_state=RS, verbose=0, warm_start=False, class_weight=None),
{'n_estimators':[25, 50, 100, 150, 200], 'max_features':['sqrt', 'log2', None], },
{'n_estimators':150, 'max_features':'log2'}), 
('SGD_l_l2_opt_a1m5',
SGDClassifier(loss='log', penalty='l2', alpha=1e-05, l1_ratio=0.15,
fit_intercept=True, max_iter=1000, tol=1e-03, shuffle=True,
verbose=0, epsilon=0.1, n_jobs=model_njobs, random_state=RS,
learning_rate='optimal', eta0=0.0, power_t=0.5, class_weight=None,
warm_start=False, average=False, n_iter=None),
{'loss': ['hinge','log','modified_huber','squared_hinge','perceptron',],
'alpha': [1.0e-6,1.0e-5, 1.0e-4, 1.0e-3, 1.0e-2], 'learning_rate': ['optimal']},
{'loss': 'log', 'learning_rate': 'optimal', 'alpha': 1e-05}), 
('LinSVC', LinearSVC() , {}, {}), 
('ABC',
AdaBoostClassifier(base_estimator=None, n_estimators=50, learning_rate=1.0,
algorithm='SAMME.R', random_state=RS),
{'n_estimators':[25, 50, 100, 150, 200], 'algorithm': ['SAMME.R', 'SAMME']}, {}), 
('GBC',
GradientBoostingClassifier(loss='deviance', learning_rate=0.1, n_estimators=100, subsample=1.0,
criterion='friedman_mse', min_samples_split=2, max_depth=3,
min_impurity_decrease=0.0, min_impurity_split=None, init=None,
random_state=RS, max_features=None, verbose=0, max_leaf_nodes=None,
warm_start=False, presort='auto', 
),
{'loss':['deviance','exponential'], 'n_estimators':[100, 150, 200, 300], 'max_features':['sqrt', 'log2', None]},
{}), 
('Perceptron_50', Perceptron(penalty=None, alpha=0.0001, fit_intercept=True, max_iter=None, tol=1e-4, shuffle=True,
verbose=0, eta0=1.0, n_jobs=model_njobs, random_state=RS, class_weight=None,
warm_start=False, n_iter=None),
{'penalty':['l2', 'l1', 'elasticnet', None], 'max_iter':[10,100,500,1000], 'eta0':[0.5,1.0,1.5]}, {}), 
('KNC_50', KNeighborsClassifier(n_neighbors=100, weights='uniform', algorithm='auto', leaf_size=30,
p=2, metric='minkowski', metric_params=None, n_jobs=model_njobs),
{'weights':['distance', 'uniform'], 'algorithm':['ball_tree', 'kd_tree', 'brute'], 'n_neighbors': [10,20,30,50,70],
'leaf_size': [10,15,20,30]},
{}), 
('LinearSVC_l2_dF', LinearSVC(penalty='l2', loss='squared_hinge', dual=False, tol=1e-4, C=1.0, multi_class='ovr',
fit_intercept=True, intercept_scaling=1, class_weight=None, verbose=0, random_state=RS, max_iter=2000),
{'dual':[True, False], 'C': [0.1, 0.25, 0.4, 0.5, 0.6, 0.75, 1.0, 1.25, 1.5, 2, 3, 4]},{}), 
('SVC_kg', SVC(C=200, kernel='rbf', degree=3,gamma=1, coef0=1, shrinking=True,tol=0.001, probability=False,
cache_size=200,class_weight=None, verbose=False, max_iter=-1,decision_function_shape=None,random_state=RS), {},{}), 
('RFC',
ExtraTreesClassifier(n_estimators=50, criterion='gini', max_depth=None, min_samples_split=2,
max_features=None, max_leaf_nodes=None, min_impurity_decrease=0.0,
min_impurity_split=None, bootstrap=False, oob_score=False, n_jobs=model_njobs, random_state=RS, verbose=0,
warm_start=False, class_weight=None),
{'max_features':['sqrt', 'log2', None],'n_estimators':[100, 500],'min_samples_leaf': [10,50,100,200,500]}, {}), 
('XGB',
XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bytree=1, gamma=1, learning_rate=0.01, max_delta_step=0,
max_depth=12, min_child_weight=1, missing=None, n_estimators=1000,
n_jobs=2, nthread=None, objective='multi:softprob', random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=True, subsample=0.8),
{}, {}),
]<train_model>
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = Input(input_dim)
 encoded = BatchNormalization()(i)
 encoded = GaussianNoise(noise )(encoded)
 encoded = Dense(640,activation='relu' )(encoded)
 decoded = Dropout(0.2 )(encoded)
 decoded = Dense(input_dim,name='decoded' )(decoded)
 x = Dense(320,activation='relu' )(decoded)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = Model(inputs=i,outputs=encoded)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=Adam(0.001),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder
if True:
SEL_MODEL = 0
model_name, model, gs_params, best_params = model_list[SEL_MODEL]
model, model_name, y_pred_valid = do_train_fit(model, model_name, True )<train_model>
def create_model(hp,input_dim,output_dim,encoder):
inputs = Input(input_dim)
 x = encoder(inputs)
 x = Concatenate()([x,inputs])
 x = BatchNormalization()(x)
 x = Dropout(hp.Float('init_dropout',0.0,0.5))(x)
 for i in range(hp.Int('num_layers',1,5)) :
x = Dense(hp.Int('num_units_{i}',128,256))(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(hp.Float(f'dropout_{i}',0.0,0.5))(x)
 x = Dense(output_dim,activation='sigmoid' )(x)
 model = Model(inputs=inputs,outputs=x)
 model.compile(optimizer=Adam(hp.Float('lr',0.00001,0.1,default=0.001)) ,loss=BinaryCrossentropy(label_smoothing=hp.Float('label_smoothing',0.0,0.1)) ,metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model
RUNMODE = 'VOTE_S'
SEL_MODEL = 0
USECLASSWEIGHTS = True
USECLASSWEIGHTS = False
USEOVR = True
print("features: ", X_train.shape)
 starttime = time.time()
model_name, model, gs_params, best_params = model_list[SEL_MODEL]
if 'class_weight' in model.__doc__ and USECLASSWEIGHTS:
model_name = 'CW_'+model_name
model.__setattr__('class_weight', class_weights)
 if RUNMODE.split('_')[0] == 'DOSCAN':
if RUNMODE[-1]=='R':clf = RandomizedSearchCV(model, param_distributions=gs_params, n_iter=20, verbose=10, n_jobs=-1)
 elif RUNMODE[-1]=='G':clf = GridSearchCV(model, gs_params, cv=3, verbose=10, n_jobs=-1)
 clf.fit(X_train, y_train)
 print('best score: ', clf.best_score_)
 print("best params: ", clf.best_params_)
 print('best est: ', clf.best_estimator_)
 with open("bestparams.txt", "w")as data:
data.write(str(clf.best_params_))
elif RUNMODE == 'KFOLD':
kfold = StratifiedKFold(n_splits=5, random_state=random_state ).split(X_train, y_train)
 scores = []
for k,(train, test)in enumerate(kfold):
training_data = X_train[train]
training_data.sort_indices()
model.fit(training_data, y_train[train])
 score = model.score(X_train[test], y_train[test])
 scores.append(score)
 print('Fold: %2d, Acc: %.3f' %(k+1, score))
elif RUNMODE == 'FIT':
model, model_name, y_pred_valid = do_train_fit(model, model_name, USEOVR)
 elif RUNMODE == 'FIT_BOTH':
model, model_name, y_pred_valid = do_train_fit(model, model_name, False)
 model, model_name, y_pred_valid = do_train_fit(model, model_name, True)
 elif RUNMODE.split('_')[0] == 'VOTE':
SEL_MODELS = [0,1,3]
WEIGHTS = [5,2,2]
vote_est = []
voting = 'hard'
if RUNMODE[-1]=='S': voting = 'soft'
v_model_name = 'V' + voting[0].upper()
for _ in SEL_MODELS:
vote_est.append(( model_list[_][0], model_list[_][1]))
v_model_name += '-'+ vote_est[-1][0]
print('*'*66)
 print('* Will use total %d ests' % len(vote_est))
for x,_ in vote_est: print('* %s' % x)
 model = VotingClassifier(estimators = vote_est,
voting = voting,
weights = WEIGHTS,
n_jobs=-1)
 if USEOVR: model = OneVsRestClassifier(model, n_jobs=4)
 model.fit(X_train, y_train)
 print("Test the model on the validation data")
 y_pred_valid = model.predict(X_valid)
 print('Accuracy: %.4f' % accuracy_score(y_pred_valid, y_valid))
print('%s %s' %(tfidf_name, model_name))
print("Time: ", conv_secs(time.time() -starttime))<save_to_csv>
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 if TRAINING:
autoencoder.fit(X,(X,y),
epochs=1002,
batch_size=16384,
validation_split=0.1,
callbacks=[EarlyStopping('val_loss',patience=10,restore_best_weights=True)])
 encoder.save_weights('./encoder.hdf5')
 else:
encoder.load_weights('.. /input/running-algos-fe-for-fast-inference/encoder.hdf5')
 encoder.trainable = False
print("Predict on submision data...")
 y_submission = model.predict(X_submission)
 y_pred = lb.inverse_transform(y_submission)
 print("Generate Submission File...")
 submission_id = [doc['id'] for doc in submission]
sub = pd.DataFrame({'id': submission_id, 'cuisine': y_pred}, columns=['id', 'cuisine'])
 sub.to_csv('%s_%s_%s.csv' %(tfidf_name, model_name, str(random_state)) , index=False )<train_model>
ENV_REAL = True
if(not TRAINING)&(not ENV_REAL):
test_col = pd.read_pickle(f'.. /input/dummy-environnement/columns_df_test.pickle')
 n_row = 15219
dummy_df = train.iloc[:n_row]
for(index, row)in tqdm(dummy_df.iterrows()):
time.sleep(0.009)
 test_df = pd.DataFrame(row ).transpose() [test_col]
test_df = pd.DataFrame(row ).transpose()
pred_df = pd.DataFrame(columns=['action'], index = [index])
 pred_df.action = 0
if False:
im = -1
exclude_ids = [0,10]
for model_name, model, gs_params, best_params in model_list:
im+=1
if im in exclude_ids: continue
try:
print('*'*66)
 print(im, model_name)
 try:model, model_name, y_pred_valid = do_train_fit(model, model_name, True)
 except:pass
except Exception as e:
print(e)
<import_modules>
if(not TRAINING)&(ENV_REAL):
env = janestreet.make_env()
th = 0.5
EWM_5000 = RunningEWMean(WIN_SIZE = 5000,n_size = 2)
 EWM_10000 = RunningEWMean(WIN_SIZE = 10000,n_size = 2)
 EWM_20000 = RunningEWMean(WIN_SIZE = 20000,n_size = 2)
 train_FE = []
for(test_df, pred_df)in tqdm(env.iter_test()):
EWM_5000.push(np.float64(np.array(test_df[['feature_0','feature_1']])))
 EWM_10000.push(np.float64(np.array(test_df[['feature_0','feature_1']])))
 EWM_20000.push(np.float64(np.array(test_df[['feature_0','feature_1']])))
 FE = []
FE = {
'feature_0_EWM_5000' : EWM_5000.get_mean() [0][0],
'feature_1_EWM_5000' : EWM_5000.get_mean() [0][1],
'feature_0_EWM_10000' : EWM_10000.get_mean() [0][0],
'feature_1_EWM_10000' : EWM_10000.get_mean() [0][1],
'feature_0_EWM_20000' : EWM_20000.get_mean() [0][0],
'feature_1_EWM_20000' : EWM_20000.get_mean() [0][1],
}
test_df_FE = pd.concat([test_df,pd.DataFrame(FE, index=[test_df.index[0]])],axis=1)
 if test_df_FE['weight'].item() > 0:
x_tt = test_df_FE.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model.predict(x_tt)for model in models],axis=0)
 pred = np.mean(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )