Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_pretrained>
%matplotlib inline
sns.set()
device = torch.device(flags.device)
 if flags.pred_mode == "multi":
predictor = LyftMultiModel(cfg)
 else:
raise ValueError(f"[ERROR] Unexpected value flags.pred_mode={flags.pred_mode}")
 pt_path = "/kaggle/input/lyft-prediction-public-models/multi_mode_448px.pth"
print(f"Loading from {pt_path}")
 state_dict = torch.load(pt_path)
 state_dict_v2 = copy.deepcopy(state_dict)
 for key in state_dict:
new_key = key[10:]
state_dict_v2[new_key] = state_dict_v2.pop(key)
 predictor.load_state_dict(state_dict_v2)
 predictor.to(device )<predict_on_test>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_train.head()
timestamps, track_ids, coords, confs = run_prediction(predictor, test_loader )<save_to_csv>
print(df_train[df_train.Sex == 'female'].Survived.sum() /df_train[df_train.Sex == 'female'].Survived.count())
 print(df_train[df_train.Sex == 'male'].Survived.sum() /df_train[df_train.Sex == 'male'].Survived.count() )
csv_path = "submission.csv"
write_pred_csv(
 csv_path,
timestamps=timestamps,
track_ids=track_ids,
coords=coords,
confs=confs)
 print(f"Saved to {csv_path}" )<install_modules>
data['Age'] = data.Age.fillna(data.Age.median())
 data['Fare'] = data.Fare.fillna(data.Fare.median())
 data.info()
!pip install pytorch-pfn-extras==0.3.1<set_options>
data = pd.get_dummies(data, columns=['Sex'], drop_first=True)
 data.head()
py.init_notebook_mode(connected=True)
 pio.templates.default = "plotly_dark"
pd.set_option('max_columns', 50 )<import_modules>
data_train = data.iloc[:891]
data_test = data.iloc[891:]
X = data_train.values
test = data_test.values
y = survived_train.values
rc('animation', html='jshtml')
 print("l5kit version:", l5kit.__version__ )<import_modules>
clf = tree.DecisionTreeClassifier(max_depth=3)
 clf.fit(X, y )
import torch
from pathlib import Path
import pytorch_pfn_extras as ppe
from math import ceil
from pytorch_pfn_extras.training import IgniteExtensionsManager
from pytorch_pfn_extras.training.triggers import MinValueTrigger
from torch import nn, optim
from torch.utils.data import DataLoader
from torch.utils.data.dataset import Subset
import pytorch_pfn_extras.training.extensions as E<choose_model_class>
Y_pred = clf.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('1st_dec_tree.csv', index=False )
class LyftMultiModel(nn.Module):
def __init__(self, cfg: Dict, num_modes=3):
super().__init__()
backbone = resnet18(pretrained=True, progress=True)
 self.backbone = backbone
num_history_channels =(cfg["model_params"]["history_num_frames"] + 1)* 2
num_in_channels = 3 + num_history_channels
self.backbone.conv1 = nn.Conv2d(
 num_in_channels,
self.backbone.conv1.out_channels,
kernel_size=self.backbone.conv1.kernel_size,
stride=self.backbone.conv1.stride,
padding=self.backbone.conv1.padding,
bias=False, 
)
 backbone_out_features = 512
self.future_len = cfg["model_params"]["future_num_frames"]
num_targets = 2 * self.future_len
self.head = nn.Sequential(
 nn.Linear(in_features=backbone_out_features, out_features=4096), 
)
 self.num_preds = num_targets * num_modes
self.num_modes = num_modes
self.logit = nn.Linear(4096, out_features=self.num_preds + num_modes)
 def forward(self, x):
x = self.backbone.conv1(x)
 x = self.backbone.bn1(x)
 x = self.backbone.relu(x)
 x = self.backbone.maxpool(x)
 x = self.backbone.layer1(x)
 x = self.backbone.layer2(x)
 x = self.backbone.layer3(x)
 x = self.backbone.layer4(x)
 x = self.backbone.avgpool(x)
 x = torch.flatten(x, 1)
 x = self.head(x)
 x = self.logit(x)
 bs, _ = x.shape
pred, confidences = torch.split(x, self.num_preds, dim=1)
 pred = pred.view(bs, self.num_modes, self.future_len, 2)
 assert confidences.shape ==(bs, self.num_modes)
 confidences = torch.softmax(confidences, dim=1)
 return pred, confidences 
<load_pretrained>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 survived_train = df_train.Survived
data = pd.concat([df_train.drop(['Survived'], axis=1), df_test])
 data['Title'] = data.Name.apply(lambda x: re.search('([A-Z][a-z]+)\.', x ).group(1))
sns.countplot(x='Title', data=data);
plt.xticks(rotation=45);
def save_yaml(filepath, content, width=120):
with open(filepath, 'w')as f:
yaml.dump(content, f, width=width)
 def load_yaml(filepath):
with open(filepath, 'r')as f:
content = yaml.safe_load(f)
 return content
class DotDict(dict):

 __getattr__ = dict.get
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__ 
<predict_on_test>
data['Has_Cabin'] = ~data.Cabin.isnull()
data.drop(['Cabin', 'Name', 'PassengerId', 'Ticket'], axis=1, inplace=True)
 data.head()
def run_prediction(predictor, data_loader):
predictor.eval()
pred_coords_list = []
confidences_list = []
timestamps_list = []
track_id_list = []
with torch.no_grad() :
dataiter = tqdm(data_loader)
 for data in dataiter:
image = data["image"].to(device)
 pred, confidences = predictor(image)
 pred_coords_list.append(pred.cpu().numpy().copy())
 confidences_list.append(confidences.cpu().numpy().copy())
 timestamps_list.append(data["timestamp"].numpy().copy())
 track_id_list.append(data["track_id"].numpy().copy())
 timestamps = np.concatenate(timestamps_list)
 track_ids = np.concatenate(track_id_list)
 coords = np.concatenate(pred_coords_list)
 confs = np.concatenate(confidences_list)
 return timestamps, track_ids, coords, confs 
<init_hyperparams>
data['Age'] = data.Age.fillna(data.Age.median())
 data['Fare'] = data.Fare.fillna(data.Fare.median())
 data['Embarked'] = data['Embarked'].fillna('S')
 data.info()
cfg = {
'format_version': 4,
'model_params': {
'model_architecture': 'resnet50',
'history_num_frames': 10,
'history_step_size': 1,
'history_delta_time': 0.1,
'future_num_frames': 50,
'future_step_size': 1,
'future_delta_time': 0.1
},
'raster_params': {
'raster_size': [224, 224],
'pixel_size': [0.5, 0.5],
'ego_center': [0.25, 0.5],
'map_type': 'py_semantic',
'satellite_map_key': 'aerial_map/aerial_map.png',
'semantic_map_key': 'semantic_map/semantic_map.pb',
'dataset_meta_key': 'meta.json',
'filter_agents_threshold': 0.5
},
'train_data_loader': {
'key': 'scenes/train.zarr',
'batch_size': 12,
'shuffle': True,
'num_workers': 4
},
'valid_data_loader': {
'key': 'scenes/validate.zarr',
'batch_size': 32,
'shuffle': False,
'num_workers': 4
},
'test_data_loader': {
'key': 'scenes/test.zarr',
'batch_size': 8,
'shuffle': False,
'num_workers': 4
},
'train_params': {
'max_num_steps': 10000,
'checkpoint_every_n_steps': 5000,
}
} 
<define_variables>
data['CatAge'] = pd.qcut(data.Age, q=4, labels=False)
 data['CatFare']= pd.qcut(data.Fare, q=4, labels=False)
 data.head()
flags_dict = {
"debug": False,
"l5kit_data_folder": "/kaggle/input/lyft-motion-prediction-autonomous-vehicles",
"pred_mode": "multi",
"device": "cuda:0",
"out_dir": "results/multi_train",
"epoch": 2,
"snapshot_freq": 50,
}<load_pretrained>
data = data.drop(['Age', 'Fare','SibSp','Parch'], axis=1)
 data.head()
flags = DotDict(flags_dict)
 out_dir = Path(flags.out_dir)
 os.makedirs(str(out_dir), exist_ok=True)
 print(f"flags: {flags_dict}")
 save_yaml(out_dir / 'flags.yaml', flags_dict)
 save_yaml(out_dir / 'cfg.yaml', cfg)
 debug = flags.debug<load_pretrained>
data_dum = pd.get_dummies(data, drop_first=True)
 data_dum.head()
l5kit_data_folder = "/kaggle/input/lyft-motion-prediction-autonomous-vehicles"
os.environ["L5KIT_DATA_FOLDER"] = l5kit_data_folder
dm = LocalDataManager(None)
 print("Load dataset...")
 default_test_cfg = {
'key': 'scenes/test.zarr',
'batch_size': 32,
'shuffle': False,
'num_workers': 4
}
test_cfg = cfg.get("test_data_loader", default_test_cfg)
 rasterizer = build_rasterizer(cfg, dm)
 test_path = test_cfg["key"]
print(f"Loading from {test_path}")
 test_zarr = ChunkedDataset(dm.require(test_path)).open()
print("test_zarr", type(test_zarr))
test_mask = np.load(f"{l5kit_data_folder}/scenes/mask.npz")["arr_0"]
test_agent_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)
 test_dataset = test_agent_dataset
if debug:
test_dataset = Subset(test_dataset, np.arange(100))
test_loader = DataLoader(
 test_dataset,
shuffle=test_cfg["shuffle"],
batch_size=test_cfg["batch_size"],
num_workers=test_cfg["num_workers"],
pin_memory=True, 
)
 print(test_agent_dataset)
 print("
print("<load_pretrained>
data_train = data_dum.iloc[:891]
data_test = data_dum.iloc[891:]
X = data_train.values
test = data_test.values
y = survived_train.values
dep = np.arange(1,9)
 param_grid = {'max_depth' : dep}
clf = tree.DecisionTreeClassifier()
clf_cv = GridSearchCV(clf, param_grid=param_grid, cv=5)
 clf_cv.fit(X, y)
 print("Tuned Decision Tree Parameters: {}".format(clf_cv.best_params_))
print("Best score is {}".format(clf_cv.best_score_))
device = torch.device(flags.device)
 if flags.pred_mode == "multi":
predictor = LyftMultiModel(cfg)
 else:
raise ValueError(f"[ERROR] Unexpected value flags.pred_mode={flags.pred_mode}")
 pt_path = "/kaggle/input/lyft-resnet18-baseline/0918_predictor_full.pt"
print(f"Loading from {pt_path}")
 predictor.load_state_dict(torch.load(pt_path))
predictor.to(device )<predict_on_test>
Y_pred = clf_cv.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('dec_tree_feat_eng.csv', index=False )
timestamps, track_ids, coords, confs = run_prediction(predictor, test_loader )<save_to_csv>
logreg = LogisticRegression()
logreg.fit(X,y)
 Y_pred = logreg.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('log_reg_feat_eng.csv', index=False)
csv_path = "submission.csv"
write_pred_csv(
 csv_path,
timestamps=timestamps,
track_ids=track_ids,
coords=coords,
confs=confs)
 print(f"Saved to {csv_path}" )<normalization>
c_space = np.logspace(-5, 8, 15)
 param_grid = {'C': c_space, 'penalty': ['l1', 'l2']}
logreg_cv = GridSearchCV(logreg,param_grid,cv=5)
 logreg_cv.fit(X,y)
 print("Tuned Logistic Regression Parameter: {}".format(logreg_cv.best_params_))
print("Tuned Logistic Regression Accuracy: {}".format(logreg_cv.best_score_))
Y_pred = logreg_cv.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('log_reg__feat_eng.csv', index=False )
random.seed(2016)
 def create_feature_map(features):
outfile = open('xgb.fmap', 'w')
 for i, feat in enumerate(features):
outfile.write('{0}\t{1}\tq
'.format(i, feat))
outfile.close()
def get_importance(gbm, features):
create_feature_map(features)
 importance = gbm.get_fscore(fmap='xgb.fmap')
 importance = sorted(importance.items() , key=itemgetter(1), reverse=True)
 return importance
def intersect(a, b):
return list(set(a)& set(b))
def run_single(train, test, features, target, random_state=0):
eta = 0.2
max_depth = 5
subsample = 0.8
colsample_bytree = 0.8
start_time = time.time()
print('XGBoost params.ETA: {}, MAX_DEPTH: {}, SUBSAMPLE: {}, COLSAMPLE_BY_TREE: {}'.format(eta, max_depth, subsample, colsample_bytree))
params = {
"objective": "binary:logistic",
"booster" : "gbtree",
"eval_metric": "auc",
"eta": eta,
"tree_method": 'exact',
"max_depth": max_depth,
"subsample": subsample,
"colsample_bytree": colsample_bytree,
"silent": 1,
"seed": random_state,
}
num_boost_round = 115
early_stopping_rounds = 10
test_size = 0.1
X_train, X_valid = train_test_split(train, test_size=test_size, random_state=random_state)
 print('Length train:', len(X_train.index))
print('Length valid:', len(X_valid.index))
y_train = X_train[target]
y_valid = X_valid[target]
dtrain = xgb.DMatrix(X_train[features], y_train)
 dvalid = xgb.DMatrix(X_valid[features], y_valid)
 watchlist = [(dtrain, 'train'),(dvalid, 'eval')]
gbm = xgb.train(params, dtrain, num_boost_round, evals=watchlist, early_stopping_rounds=early_stopping_rounds, verbose_eval=True)
 print("Validating...")
 check = gbm.predict(xgb.DMatrix(X_valid[features]), ntree_limit=gbm.best_iteration+1)
 score = roc_auc_score(X_valid[target].values, check)
 print('Check error value: {:.6f}'.format(score))
imp = get_importance(gbm, features)
 print('Importance array: ', imp)
 print("Predict test set...")
 test_prediction = gbm.predict(xgb.DMatrix(test[features]), ntree_limit=gbm.best_iteration+1)
 print('Training time: {} minutes'.format(round(( time.time() - start_time)/60, 2)))
 return test_prediction.tolist() , score
def run_kfold(nfolds, train, test, features, target, random_state=0):
eta = 0.2
max_depth = 5
subsample = 0.8
colsample_bytree = 0.8
start_time = time.time()
print('XGBoost params.ETA: {}, MAX_DEPTH: {}, SUBSAMPLE: {}, COLSAMPLE_BY_TREE: {}'.format(eta, max_depth, subsample, colsample_bytree))
params = {
"objective": "binary:logistic",
"booster" : "gbtree",
"eval_metric": "auc",
"eta": eta,
"max_depth": max_depth,
"subsample": subsample,
"colsample_bytree": colsample_bytree,
"silent": 1,
"seed": random_state
}
num_boost_round = 150
early_stopping_rounds = 10
yfull_train = dict()
yfull_test = copy.deepcopy(test[['activity_id']].astype(object))
kf = KFold(len(train.index), n_folds=nfolds, shuffle=True, random_state=random_state)
 num_fold = 0
for train_index, test_index in kf:
num_fold += 1
print('Start fold {} from {}'.format(num_fold, nfolds))
X_train, X_valid = train[features].as_matrix() [train_index], train[features].as_matrix() [test_index]
y_train, y_valid = train[target].as_matrix() [train_index], train[target].as_matrix() [test_index]
X_test = test[features].as_matrix()
print('Length train:', len(X_train))
print('Length valid:', len(X_valid))
dtrain = xgb.DMatrix(X_train, y_train)
 dvalid = xgb.DMatrix(X_valid, y_valid)
 watchlist = [(dtrain, 'train'),(dvalid, 'eval')]
gbm = xgb.train(params, dtrain, num_boost_round, evals=watchlist, early_stopping_rounds=early_stopping_rounds, verbose_eval=True)
 print("Validating...")
 yhat = gbm.predict(xgb.DMatrix(X_valid), ntree_limit=gbm.best_iteration+1)
 score = roc_auc_score(y_valid.tolist() , yhat)
 print('Check error value: {:.6f}'.format(score))
for i in range(len(test_index)) :
yfull_train[test_index[i]] = yhat[i]
imp = get_importance(gbm, features)
 print('Importance array: ', imp)
 print("Predict test set...")
 test_prediction = gbm.predict(xgb.DMatrix(X_test), ntree_limit=gbm.best_iteration+1)
 yfull_test['kfold_' + str(num_fold)] = test_prediction
train_res = []
for i in sorted(yfull_train.keys()):
train_res.append(yfull_train[i])
 score = roc_auc_score(train[target], np.array(train_res))
print('Check error value: {:.6f}'.format(score))
merge = []
for i in range(1, nfolds+1):
merge.append('kfold_' + str(i))
yfull_test['mean'] = yfull_test[merge].mean(axis=1)
 print('Training time: {} minutes'.format(round(( time.time() - start_time)/60, 2)))
 return yfull_test['mean'].values, score
def create_submission(score, test, prediction):
now = datetime.datetime.now()
sub_file = 'submission_' + str(score)+ '_' + str(now.strftime("%Y-%m-%d-%H-%M")) + '.csv'
print('Writing submission: ', sub_file)
 f = open(sub_file, 'w')
 f.write('activity_id,outcome
')
 total = 0
for id in test['activity_id']:
str1 = str(id)+ ',' + str(prediction[total])
 str1 += '
'
total += 1
f.write(str1)
 f.close()
def get_features(train, test):
trainval = list(train.columns.values)
 testval = list(test.columns.values)
 output = intersect(trainval, testval)
 output.remove('people_id')
 output.remove('activity_id')
 return sorted(output)
 def read_test_train() :
print("Read people.csv...")
 people = pd.read_csv(".. /input/people.csv",
dtype={'people_id': np.str,
'activity_id': np.str,
'char_38': np.int32},
parse_dates=['date'])
 print("Load train.csv...")
 train = pd.read_csv(".. /input/act_train.csv",
dtype={'people_id': np.str,
'activity_id': np.str,
'outcome': np.int8},
parse_dates=['date'])
 print("Load test.csv...")
 test = pd.read_csv(".. /input/act_test.csv",
dtype={'people_id': np.str,
'activity_id': np.str},
parse_dates=['date'])
 print("Process tables...")
 for table in [train, test]:
table['year'] = table['date'].dt.year
table['month'] = table['date'].dt.month
table['day'] = table['date'].dt.day
table.drop('date', axis=1, inplace=True)
 table['activity_category'] = table['activity_category'].str.lstrip('type ' ).astype(np.int32)
 for i in range(1, 11):
table['char_' + str(i)].fillna('type -999', inplace=True)
 table['char_' + str(i)] = table['char_' + str(i)].str.lstrip('type ' ).astype(np.int32)
 people['year'] = people['date'].dt.year
people['month'] = people['date'].dt.month
people['day'] = people['date'].dt.day
people['weekday'] = people['date'].dt.weekday
people['weekend'] =(( people.weekday == 0)|(people.weekday == 6)).astype(int)
 people.drop('date', axis=1, inplace=True)
 people['group_1'] = people['group_1'].str.lstrip('group ' ).astype(np.int32)
 for i in range(1, 10):
people['char_' + str(i)] = people['char_' + str(i)].str.lstrip('type ' ).astype(np.int32)
 for i in range(10, 38):
people['char_' + str(i)] = people['char_' + str(i)].astype(np.int32)
 print("Merge...")
 train = pd.merge(train, people, how='left', on='people_id', left_index=True)
 train.fillna(-999, inplace=True)
 test = pd.merge(test, people, how='left', on='people_id', left_index=True)
 test.fillna(-999, inplace=True)
 features = get_features(train, test)
 return train, test, features
train, test, features = read_test_train()
print('Length of train: ', len(train))
print('Length of test: ', len(test))
print('Features [{}]: {}'.format(len(features), sorted(features)))
 test_prediction, score = run_single(train, test, features, 'outcome')
 print(type(test_prediction))
create_submission(score, test, test_prediction )<load_from_csv>
rf_clf = RandomForestClassifier()
rf_clf.fit(X,y)
 Y_pred = rf_clf.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('random_forest_feat_eng.csv', index=False )
items=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/items.csv")
 shops=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/shops.csv")
 cats=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/item_categories.csv")
 train=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/sales_train.csv")
 test=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/test.csv" )<filter>
n_estimators = np.arange(10,50)
 params_grid = {'n_estimators':n_estimators}
rf_clf = RandomForestClassifier()
rf_clf_cv = GridSearchCV(rf_clf,params_grid,cv=5)
 rf_clf_cv.fit(X,y)
 print("Tuned Random Forest Classifier Parameter: {}".format(rf_clf_cv.best_params_))
print("Tuned Random Forest Classifier Accuracy: {}".format(rf_clf_cv.best_score_))
Y_pred = rf_clf_cv.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('random_forest_cv_feat_eng.csv', index=False )
train = train[(train.item_price < 300000)&(train.item_cnt_day < 1000)]<feature_engineering>
knn = KNeighborsClassifier()
knn.fit(X,y)
 Y_pred = knn.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('knn_feat_eng.csv', index=False )
<feature_engineering><EOS> 
n_neighbors = np.arange(1,20)
 params_grid = {'n_neighbors':n_neighbors}
knn = KNeighborsClassifier()
knn_cv = GridSearchCV(knn,params_grid,cv=5)
 knn_cv.fit(X,y)
 print("Tuned KNN Classifier Parameter: {}".format(knn_cv.best_params_))
print("Tuned KNN Classifier Accuracy: {}".format(knn_cv.best_score_))
Y_pred = knn_cv.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('knn_cv_feat_eng.csv', index=False )