Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
pd.options.mode.chained_assignment = None
sub_col = data1['id']<create_dataframe>
train_set = pd.read_csv('.. /input/train.csv')
 test_set = pd.read_csv('.. /input/test.csv' )
all_cols = pd.DataFrame({})
 all_cols['id'] = sub_col
all_cols[categories] = 0.60*data1[categories] + 0.40*data2[categories]<save_to_csv>
train_set.isnull().sum()
all_cols.to_csv('sub.csv', index=False )<set_options>
test_set.isnull().sum()
pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 500)
 warnings.filterwarnings('ignore' )<define_variables>
fare_set = train_set[['Fare','Survived']].copy()
cont_plot(fare_set, 'Fare', 'Survived', survival_palette, [1, 0], range(0,550,50))
INPUT_DIR_PATH = '.. /input/m5-forecasting-accuracy/'<split>
combined_set = [train_set, test_set]
for dataset in combined_set:
dataset["Age"].fillna(dataset["Age"].median() , inplace=True)
 dataset["Fare"].fillna(dataset["Fare"].median() , inplace=True)
 train_set["Embarked"].fillna(train_set["Embarked"].value_counts().index[0], inplace=True )
sell_prices_df, calendar_df, sales_train_validation_df, submission_df = read_data()<define_variables>
train_set.isnull().sum()
NUM_ITEMS = sales_train_validation_df.shape[0]
DAYS_PRED = 28
nrows = 365 * 2 * NUM_ITEMS<categorify>
test_set.isnull().sum()
def encode_categorical(df, cols):
for col in cols:
le = preprocessing.LabelEncoder()
not_null = df[col][df[col].notnull() ]
df[col] = pd.Series(le.fit_transform(not_null), index=not_null.index)
 return df
calendar_df = encode_categorical(calendar_df, ["event_name_1", "event_type_1", "event_name_2", "event_type_2"] ).pipe(reduce_mem_usage)
 sales_train_validation_df = encode_categorical(sales_train_validation_df, ["item_id", "dept_id", "cat_id", "store_id", "state_id"] ).pipe(reduce_mem_usage)
 sell_prices_df = encode_categorical(sell_prices_df, ["item_id", "store_id"] ).pipe(reduce_mem_usage )<categorify>
for dataset in combined_set:
dataset['Family'] = ''
dataset.loc[dataset['FamilySize'] == 0, 'Family'] = 'alone'
dataset.loc[(dataset['FamilySize'] > 0)&(dataset['FamilySize'] <= 3), 'Family'] = 'small'
dataset.loc[(dataset['FamilySize'] > 3)&(dataset['FamilySize'] <= 6), 'Family'] = 'medium'
dataset.loc[dataset['FamilySize'] > 6, 'Family'] = 'large'
def melt_and_merge(calendar, sell_prices, sales_train_validation, submission, nrows = 55000000, merge = False):
sales_train_validation = pd.melt(sales_train_validation, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'], var_name = 'day', value_name = 'demand')
 print('Melted sales train validation has {} rows and {} columns'.format(sales_train_validation.shape[0], sales_train_validation.shape[1]))
sales_train_validation = reduce_mem_usage(sales_train_validation)
 sales_train_validation = sales_train_validation.iloc[-nrows:,:]
test1_rows = [row for row in submission['id'] if 'validation' in row]
test2_rows = [row for row in submission['id'] if 'evaluation' in row]
test1 = submission[submission['id'].isin(test1_rows)]
test2 = submission[submission['id'].isin(test2_rows)]
test1.columns = ['id', 'd_1914', 'd_1915', 'd_1916', 'd_1917', 'd_1918', 'd_1919', 'd_1920', 'd_1921', 'd_1922', 'd_1923', 'd_1924', 'd_1925', 'd_1926', 'd_1927', 'd_1928', 'd_1929', 'd_1930', 'd_1931',
'd_1932', 'd_1933', 'd_1934', 'd_1935', 'd_1936', 'd_1937', 'd_1938', 'd_1939', 'd_1940', 'd_1941']
test2.columns = ['id', 'd_1942', 'd_1943', 'd_1944', 'd_1945', 'd_1946', 'd_1947', 'd_1948', 'd_1949', 'd_1950', 'd_1951', 'd_1952', 'd_1953', 'd_1954', 'd_1955', 'd_1956', 'd_1957', 'd_1958', 'd_1959',
'd_1960', 'd_1961', 'd_1962', 'd_1963', 'd_1964', 'd_1965', 'd_1966', 'd_1967', 'd_1968', 'd_1969']
product = sales_train_validation[['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']].drop_duplicates()
test2['id'] = test2['id'].str.replace('_evaluation','_validation')
 test1 = test1.merge(product, how = 'left', on = 'id')
 test2 = test2.merge(product, how = 'left', on = 'id')
 test2['id'] = test2['id'].str.replace('_validation','_evaluation')
 test1 = pd.melt(test1, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'], var_name = 'day', value_name = 'demand')
 test2 = pd.melt(test2, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'], var_name = 'day', value_name = 'demand')
 sales_train_validation['part'] = 'train'
test1['part'] = 'test1'
test2['part'] = 'test2'
data = pd.concat([sales_train_validation, test1, test2], axis = 0)
 del sales_train_validation, test1, test2
print(data.shape)
 calendar.drop(['weekday', 'wday', 'month', 'year'], inplace = True, axis = 1)
 data = data[data['part'] != 'test2']
if merge:
data = pd.merge(data, calendar, how = 'left', left_on = ['day'], right_on = ['d'])
 data.drop(['d', 'day'], inplace = True, axis = 1)
 data = data.merge(sell_prices, on = ['store_id', 'item_id', 'wm_yr_wk'], how = 'left')
 print('Our final dataset to train has {} rows and {} columns'.format(data.shape[0], data.shape[1]))
else:
pass
gc.collect()
return data<create_dataframe>
title_dict = {
"Mr" : "Mr",
"Miss" : "Miss",
"Mrs" : "Mrs",
"Master" : "Master",
"Dr": "Scholar",
"Rev": "Religious",
"Col": "Officer",
"Major": "Officer",
"Mlle": "Miss",
"Don": "Noble",
"the Countess":"Noble",
"Ms": "Mrs",
"Mme": "Mrs",
"Capt": "Noble",
"Lady" : "Noble",
"Sir" : "Noble",
"Jonkheer": "Noble"
}
for dataset in combined_set:
dataset['TitleGroup'] = dataset.Title.map(title_dict )
nrows = 27500000
data = melt_and_merge(calendar_df, sell_prices_df, sales_train_validation_df, submission_df, nrows = nrows, merge = True)
<categorify>
print(test_set[test_set['TitleGroup'].isnull() == True] )
def transform(data):
nan_features = ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for feature in nan_features:
data[feature].fillna('unknown', inplace = True)
 cat = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for feature in cat:
encoder = preprocessing.LabelEncoder()
data[feature] = encoder.fit_transform(data[feature])
 return data
def simple_fe(data):
for val in [28, 29, 30]:
data[f"shift_t{val}"] = data.groupby(["id"])["demand"].transform(lambda x: x.shift(val))
for val in [7, 30, 60, 90, 180]:
data[f"rolling_std_t{val}"] = data.groupby(["id"])["demand"].transform(lambda x: x.shift(28 ).rolling(val ).std())
 for val in [7, 30, 60, 90, 180]:
data[f"rolling_mean_t{val}"] = data.groupby(["id"])["demand"].transform(lambda x: x.shift(28 ).rolling(val ).mean())
 data["rolling_skew_t30"] = data.groupby(["id"])["demand"].transform(lambda x: x.shift(28 ).rolling(30 ).skew())
 data["rolling_kurt_t30"] = data.groupby(["id"])["demand"].transform(lambda x: x.shift(28 ).rolling(30 ).kurt())
 data['lag_price_t1'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1))
data['price_change_t1'] =(data['lag_price_t1'] - data['sell_price'])/(data['lag_price_t1'])
 data['rolling_price_max_t365'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1 ).rolling(365 ).max())
 data['price_change_t365'] =(data['rolling_price_max_t365'] - data['sell_price'])/(data['rolling_price_max_t365'])
 data['rolling_price_std_t7'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(7 ).std())
 data['rolling_price_std_t30'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(30 ).std())
 data.drop(['rolling_price_max_t365', 'lag_price_t1'], inplace = True, axis = 1)
 data['date'] = pd.to_datetime(data['date'])
 attrs = ["year", "quarter", "month", "week", "day", "dayofweek", "is_year_end", "is_year_start", "is_quarter_end", \
"is_quarter_start", "is_month_end","is_month_start",
]
for attr in attrs:
dtype = np.int16 if attr == "year" else np.int8
data[attr] = getattr(data['date'].dt, attr ).astype(dtype)
 data["is_weekend"] = data["dayofweek"].isin([5, 6] ).astype(np.int8)
 return data 
<define_variables>
test_set.at[414, 'TitleGroup'] = 'Noble'
features = [
"item_id", "dept_id", "cat_id", "store_id", "state_id", "event_name_1", "event_type_1", "snap_CA", "snap_TX", \
"snap_WI", "sell_price", \
"shift_t28", "rolling_std_t7", "rolling_std_t30", "rolling_std_t90", "rolling_std_t180", \
"rolling_mean_t7", "rolling_mean_t30", "rolling_mean_t60", \
"price_change_t1", "price_change_t365", "rolling_price_std_t7",
"year", "month", "dayofweek",
] 
("wday", "month", "year",
"event_name_1", "event_type_1",
"snap_CA", "snap_TX", "snap_WI",
"sell_price", "sell_price_rel_diff", "sell_price_cumrel", "sell_price_roll_sd7",
"lag_t28", "rolling_mean_t7", "rolling_mean_t30", "rolling_mean_t60",
"rolling_mean_t90", "rolling_mean_t180", "rolling_sd_t7", "rolling_sd_t30",
"item_id", "dept_id", "cat_id", "store_id", "state_id" )<prepare_x_and_y>
X_train = train_set.drop(['Survived','PassengerId','Name','Age','Fare','Ticket','Cabin','SibSp','Parch','Title','FamilySize'], axis=1)
 X_test = test_set.drop(['PassengerId','Name','Age','Fare','Ticket','Cabin','SibSp','Parch','Title','FamilySize'], axis=1)
 y_train = train_set['Survived']
def run_lgb(data):
x_train = data[data['date'] <= '2016-03-27']
y_train = x_train['demand']
x_val = data[(data['date'] > '2016-03-27')&(data['date'] <= '2016-04-24')]
y_val = x_val['demand']
test = data[(data['date'] > '2016-04-24')]
del data
gc.collect()
params = {
'metric': 'rmse',
'objective': 'poisson',
'n_jobs': -1,
'seed': 20,
'learning_rate': 0.1,
'alpha': 0.1,
'lambda': 0.1,
'bagging_fraction': 0.66,
'bagging_freq': 2,
'colsample_bytree': 0.77}
train_set = lgb.Dataset(x_train[features], y_train)
 val_set = lgb.Dataset(x_val[features], y_val)
 del x_train, y_train
model = lgb.train(params, train_set, num_boost_round = 2000, early_stopping_rounds = 200, valid_sets = [train_set, val_set], verbose_eval = 100)
 joblib.dump(model, 'lgbm_0.sav')
 val_pred = model.predict(x_val[features], num_iteration=model.best_iteration)
 val_score = np.sqrt(metrics.mean_squared_error(val_pred, y_val))
print(f'Our val rmse score is {val_score}')
 y_pred = model.predict(test[features], num_iteration=model.best_iteration)
 test['demand'] = y_pred
return test
def predict(test, submission):
predictions = test[['id', 'date', 'demand']]
predictions = pd.pivot(predictions, index = 'id', columns = 'date', values = 'demand' ).reset_index()
predictions.columns = ['id'] + ['F' + str(i + 1)for i in range(28)]
evaluation_rows = [row for row in submission['id'] if 'evaluation' in row]
evaluation = submission[submission['id'].isin(evaluation_rows)]
validation = submission[['id']].merge(predictions, on = 'id')
 final = pd.concat([validation, evaluation])
 final.to_csv('submission.csv', index = False)
 def transform_train_and_eval(data):
data = simple_fe(data)
 data = reduce_mem_usage(data)
 test = run_lgb(data)
 predict(test, submission_df)
<categorify>
X_train_analysis = X_train.copy()
X_train_analysis['Sex'] = X_train_analysis['Sex'].map({'male': 0, 'female': 1} ).astype(int)
 X_train_analysis['Embarked'] = X_train_analysis['Embarked'].map({'C': 0, 'Q': 1, 'S': 2} ).astype(int)
 X_train_analysis['Family'] = X_train_analysis['Family'].map({'alone': 0, 'small': 1, 'medium': 2, 'large': 3} ).astype(int)
 agerange_dict = dict(zip(age_labels, list(range(len(age_labels)))))
 X_train_analysis['AgeRange'] = X_train_analysis['AgeRange'].map(agerange_dict ).astype(int)
 farerange_dict = dict(zip(fare_labels, list(range(len(fare_labels)))))
 X_train_analysis['FareRange'] = X_train_analysis['FareRange'].map(farerange_dict ).astype(int)
 titlegroup_labels = list(set(title_dict.values()))
titlegroup_dict = dict(zip(titlegroup_labels, list(range(len(titlegroup_labels)))))
 X_train_analysis['TitleGroup'] = X_train_analysis['TitleGroup'].map(titlegroup_dict ).astype(int )
transform_train_and_eval(data )<install_modules>
rforest_checker = RandomForestClassifier(random_state = 0)
 rforest_checker.fit(X_train_analysis, y_train)
 importances_df = pd.DataFrame(rforest_checker.feature_importances_, columns=['Feature_Importance'],
index=X_train_analysis.columns)
 importances_df.sort_values(by=['Feature_Importance'], ascending=False, inplace=True)
 print(importances_df )
!pip install bert-for-tf2
!pip install sentencepiece<set_options>
my_imp_dict = {'Feature Importance' : pd.Series([0.360313, 0.113686, 0.109495, 0.103845, 0.100966, 0.099818, 0.056429, 0.055449],
index=['TitleGroup', 'Family', 'Pclass', 'Sex','FareRange', 'AgeRange', 'HasCabin', 'Embarked'])}
my_imp_df = pd.DataFrame(my_imp_dict)
 print(my_imp_df )
try:
%tensorflow_version 2.x
except Exception:
pass 
<feature_engineering>
X_train = X_train.drop(['HasCabin','Embarked'], axis=1)
 X_test = X_test.drop(['HasCabin','Embarked'], axis=1 )
FullTokenizer = bert.bert_tokenization.FullTokenizer
bert_layer = hub.KerasLayer("https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1", trainable=False)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = FullTokenizer(vocab_file, do_lower_case )<load_from_csv>
X_train = pd.get_dummies(X_train, columns=['TitleGroup','Family','Pclass','Sex','AgeRange','FareRange'])
 X_test = pd.get_dummies(X_test, columns=['TitleGroup','Family','Pclass','Sex','AgeRange','FareRange'] )
train_cols = ["id", "keyword", "location", "text", "target"]
train = pd.read_csv(
 "/kaggle/input/nlp-getting-started/train.csv",
header=None,
names=train_cols,
skiprows=1,
engine="python",
encoding="latin1" 
)
 test_cols = ["id", "keyword", "location", "text"]
test = pd.read_csv(
 "/kaggle/input/nlp-getting-started/test.csv",
header=None,
names=test_cols,
skiprows=1,
engine="python",
encoding="latin1" 
)
 def fix_keyword(x):
return str(x[1] ).replace('%20', ' ')
 def new_text(x):
return str(x[1])+ ' ' + str(x[2])+ ' ' +str(x[3])
 def clean_tweet(tweet):
tweet = re.sub(r"https?://[A-Za-z0-9./]+", ' ', tweet)
 tweet = re.sub(r"@[A-Za-z0-9]+", ' ', tweet)
 tweet = re.sub(r"[^a-zA-Z0-9!?']", ' ', tweet)
 tweet = re.sub(r" +", ' ', tweet)
 tweet = re.sub(r"\?+", ' Q', tweet)
 tweet = re.sub(r"\!+", ' X', tweet)
 return tweet
def encode_sentence(sent):
return ["[CLS]"] + tokenizer.tokenize(sent)+ ["[SEP]"]
train.keyword = train.apply(fix_keyword, axis=1)
 train['new_text'] = train.apply(new_text, axis=1)
 train_clean = [clean_tweet(tweet)for tweet in train.new_text]
train_inputs = [encode_sentence(sentence)for sentence in train_clean]
train_labels = train.target.values
test.keyword = test.apply(fix_keyword, axis=1)
 test['new_text'] = test.apply(new_text, axis=1)
 test_clean = [clean_tweet(tweet)for tweet in test.new_text]
test_inputs = [encode_sentence(sentence)for sentence in test_clean]<categorify>
X_train = X_train.drop(['Pclass_1','Sex_female','TitleGroup_Master','AgeRange_15-','FareRange_10-','Family_alone'], axis=1)
 X_test = X_test.drop(['Pclass_1','Sex_female','TitleGroup_Master','AgeRange_15-','FareRange_10-','Family_alone'], axis=1 )
def get_ids(tokens):
return tokenizer.convert_tokens_to_ids(tokens)
 def get_mask(tokens):
return np.char.not_equal(tokens, "[PAD]" ).astype(int)
 def get_segments(tokens):
seg_ids = []
current_seg_id = 0
for tok in tokens:
seg_ids.append(current_seg_id)
 if tok == "[SEP]":
current_seg_id = 1-current_seg_id
return seg_ids<randomize_order>
params_logreg = [{'C': [0.01, 0.1, 1, 10, 100], 'penalty': ['l1','l2']}]
grid_logreg = GridSearchCV(estimator = LogisticRegression() ,
param_grid = params_logreg,
scoring = 'accuracy',
cv = 10)
 grid_logreg = grid_logreg.fit(X_train, y_train)
 best_acc_logreg = grid_logreg.best_score_
best_params_logreg = grid_logreg.best_params_
data_with_len = [[sent, train_labels[i], len(sent)]
for i, sent in enumerate(train_inputs)]
random.shuffle(data_with_len)
 data_with_len.sort(key=lambda x: x[2])
 train_all = [ 
(
 [
get_ids(sent_lab[0]),
get_mask(sent_lab[0]),
get_segments(sent_lab[0])
 ],
sent_lab[1] 
)
 for sent_lab in data_with_len] 
<create_dataframe>
params_ksvm = [{'C': [0.1, 1, 10, 100], 'kernel': ['linear']},
{'C': [0.1, 1, 10, 100], 'kernel': ['rbf'],
'gamma': [0.1, 0.2, 0.3, 0.4, 0.5]},
{'C': [0.1, 1, 10, 100], 'kernel': ['poly'],
'degree': [1, 2, 3],
'gamma': [0.1, 0.2, 0.3, 0.4, 0.5]}]
grid_ksvm = GridSearchCV(estimator = SVC(random_state = 0),
param_grid = params_ksvm,
scoring = 'accuracy',
cv = 10,
n_jobs=-1)
 grid_ksvm = grid_ksvm.fit(X_train, y_train)
 best_acc_ksvm = grid_ksvm.best_score_
best_params_ksvm = grid_ksvm.best_params_
all_dataset = tf.data.Dataset.from_generator(lambda: train_all, output_types=(tf.int32, tf.int32))<define_variables>
params_dtree = [{'min_samples_split': [5, 10, 15, 20],
'min_samples_leaf': [1, 2, 3],
'max_features': ['auto', 'log2']}]
grid_dtree = GridSearchCV(estimator = DecisionTreeClassifier(criterion = 'gini',
random_state = 0),
param_grid = params_dtree,
scoring = 'accuracy',
cv = 10,
n_jobs=-1)
 grid_dtree = grid_dtree.fit(X_train, y_train)
 best_acc_dtree = grid_dtree.best_score_
best_params_dtree = grid_dtree.best_params_
BATCH_SIZE = 32
all_batched = all_dataset.padded_batch(BATCH_SIZE,
padded_shapes=(( 3, None),()),
padding_values=(0, 0))
NB_BATCHES = math.ceil(len(train_all)/ BATCH_SIZE)
 NB_BATCHES_TEST = NB_BATCHES // 10
all_batched.shuffle(NB_BATCHES)
 test_dataset = all_batched.take(NB_BATCHES_TEST)
 train_dataset = all_batched.skip(NB_BATCHES_TEST )<choose_model_class>
params_rforest = [{'n_estimators': [200, 300],
'max_depth': [5, 7, 10],
'min_samples_split': [2, 4]}]
grid_rforest = GridSearchCV(estimator = RandomForestClassifier(criterion = 'gini',
random_state = 0, n_jobs=-1),
param_grid = params_rforest,
scoring = 'accuracy',
cv = 10,
n_jobs=-1)
 grid_rforest = grid_rforest.fit(X_train, y_train)
 best_acc_rforest = grid_rforest.best_score_
best_params_rforest = grid_rforest.best_params_
class DCNNBERTEmbedding(tf.keras.Model):
def __init__(self,
nb_filters=50,
FFN_units=512,
nb_classes=2,
dropout_rate=0.1,
name="dcnn"):
super(DCNNBERTEmbedding, self ).__init__(name=name)
 self.bert_layer = hub.KerasLayer(
 "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1",
trainable=True)
 self.bigram = layers.Conv1D(filters=nb_filters,
kernel_size=2,
padding="valid",
activation="relu")
 self.trigram = layers.Conv1D(filters=nb_filters,
kernel_size=3,
padding="valid",
activation="relu")
 self.fourgram = layers.Conv1D(filters=nb_filters,
kernel_size=4,
padding="valid",
activation="relu")
 self.pool = layers.GlobalMaxPool1D()
self.dense_1 = layers.Dense(units=FFN_units, activation="relu")
 self.dropout = layers.Dropout(rate=dropout_rate)
 if nb_classes == 2:
self.last_dense = layers.Dense(units=1,
activation="sigmoid")
 else:
self.last_dense = layers.Dense(units=nb_classes,
activation="softmax")
 def embed_with_bert(self, all_tokens):
_, embs = self.bert_layer([all_tokens[:, 0, :],
all_tokens[:, 1, :],
all_tokens[:, 2, :]])
 return embs
def call(self, inputs, training):
x = self.embed_with_bert(inputs)
 x_1 = self.bigram(x)
 x_1 = self.pool(x_1)
 x_2 = self.trigram(x)
 x_2 = self.pool(x_2)
 x_3 = self.fourgram(x)
 x_3 = self.pool(x_3)
 merged = tf.concat([x_1, x_2, x_3], axis=-1)
 merged = self.dense_1(merged)
 merged = self.dropout(merged, training)
 output = self.last_dense(merged)
 return output<define_variables>
grid_score_dict = {'Best Score': [best_acc_logreg,best_acc_ksvm,best_acc_dtree,best_acc_rforest],
'Optimized Parameters': [best_params_logreg,best_params_ksvm,best_params_dtree,best_params_rforest],
}
pd.DataFrame(grid_score_dict, index=['Logistic Regression','Kernel SVM','Decision Tree','Random Forest'] )
NB_FILTERS = 128
FFN_UNITS = 256
NB_CLASSES = 2
DROPOUT_RATE = 0.2
BATCH_SIZE = 32
NB_EPOCHS = 3<choose_model_class>
logreg = LogisticRegression(C = 1, penalty = 'l1')
 logreg.fit(X_train, y_train)
 y_pred_train_logreg = cross_val_predict(logreg, X_train, y_train)
 y_pred_test_logreg = logreg.predict(X_test )
Dcnn = DCNNBERTEmbedding(nb_filters=NB_FILTERS,
FFN_units=FFN_UNITS,
nb_classes=NB_CLASSES,
dropout_rate=DROPOUT_RATE)
 if NB_CLASSES == 2:
Dcnn.compile(loss="binary_crossentropy",
optimizer=tf.optimizers.Adam(learning_rate=2e-5),
metrics=["accuracy"])
 else:
Dcnn.compile(loss="sparse_categorical_crossentropy",
optimizer="adam",
metrics=["sparse_categorical_accuracy"])
 Dcnn.fit(all_batched, epochs=NB_EPOCHS )<compute_test_metric>
ksvm = SVC(C = 1, gamma = 0.2, kernel = 'rbf', random_state = 0)
 ksvm.fit(X_train, y_train)
 y_pred_train_ksvm = cross_val_predict(ksvm, X_train, y_train)
 y_pred_test_ksvm = ksvm.predict(X_test )
results = Dcnn.evaluate(test_dataset)
 print(results )<feature_engineering>
dtree = DecisionTreeClassifier(criterion = 'gini', max_features='auto', min_samples_leaf=1, min_samples_split=5, random_state = 0)
 dtree.fit(X_train, y_train)
 y_pred_train_dtree = cross_val_predict(dtree, X_train, y_train)
 y_pred_test_dtree = dtree.predict(X_test )
cols = ["id", "keyword", "location", "text"]
test = pd.read_csv(
 "/kaggle/input/nlp-getting-started/test.csv",
header=None,
names=cols,
skiprows=1,
engine="python",
encoding="latin1" 
)
 test.keyword = test.apply(fix_keyword, axis=1)
 test['new_text'] = test.apply(new_text, axis=1)
 test_clean = [clean_tweet(tweet)for tweet in test.new_text]
test_inputs = [encode_sentence(sentence)for sentence in test_clean]<predict_on_test>
rforest = RandomForestClassifier(max_depth = 7, min_samples_split=4, n_estimators = 200, random_state = 0)
 rforest.fit(X_train, y_train)
 y_pred_train_rforest = cross_val_predict(rforest, X_train, y_train)
 y_pred_test_rforest = rforest.predict(X_test )
preds = []
for sentence in test_inputs:
input = [[
get_ids(sentence),
get_mask(sentence),
get_segments(sentence)
 ]]
preds.append(int(np.round(Dcnn.predict(input)[0][0])))
 if len(preds)% 100 == 0:
print('Predictions made:', len(preds)) 
<save_to_csv>
second_layer_train = pd.DataFrame({'Logistic Regression': y_pred_train_logreg.ravel() ,
'Kernel SVM': y_pred_train_ksvm.ravel() ,
'Decision Tree': y_pred_train_dtree.ravel() ,
'Random Forest': y_pred_train_rforest.ravel()
})
 second_layer_train.head()
X_train_second = np.concatenate(( y_pred_train_logreg.reshape(-1, 1), y_pred_train_ksvm.reshape(-1, 1),
y_pred_train_dtree.reshape(-1, 1), y_pred_train_rforest.reshape(-1, 1)) ,
axis=1)
 X_test_second = np.concatenate(( y_pred_test_logreg.reshape(-1, 1), y_pred_test_ksvm.reshape(-1, 1),
y_pred_test_dtree.reshape(-1, 1), y_pred_test_rforest.reshape(-1, 1)) ,
axis=1)
 xgb = XGBClassifier(
 n_estimators= 800,
max_depth= 4,
min_child_weight= 2,
gamma=0.9,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread= -1,
scale_pos_weight=1 ).fit(X_train_second, y_train)
 y_pred = xgb.predict(X_test_second )
<load_from_url><EOS> 
passengerId = np.array(test_set['PassengerId'] ).astype(int)
 submission = pd.DataFrame({ 'PassengerId' : passengerId, 'Survived' : y_pred })
 print(submission.shape)
 submission.to_csv('submission.csv', index=False )