Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<create_dataframe>
import numpy as np
import pandas as pd
import os
from sklearn.model_selection import train_test_split
def estimate_counts_based_on_real_testing_samples(training_and_validation_sets: List[pd.DataFrame], testing_set: pd.DataFrame, columns: List)-> List:

 real_samples_indices, _ = get_real_synthetic_testing_samples(testing_set)
 data_set_for_estimation = pd.concat([*training_and_validation_sets, testing_set.loc[real_samples_indices]])
 all_dataframes = [*[d.copy(deep=True)for d in training_and_validation_sets], testing_set.copy(deep=True)]
decimals = {
'count': 4,
'mean': 4,
'std': 4,
'sum': 4,
'min': 2,
'max': 3
}
args = []
n_processes = 10
columns = np.array(columns)
 batch_size = int(np.ceil(len(columns)/ n_processes))
for i in range(n_processes):
indices = list(range(batch_size * i, batch_size *(i + 1)))
 args.append(( data_set_for_estimation.iloc[:, indices], [dataframe.iloc[:, indices] for dataframe in all_dataframes], columns[indices], decimals))
with multiprocessing.Pool(processes=n_processes)as p:
chunks = p.starmap(add_features_in_batch, args)
 all_dataframes = [pd.concat(d, axis=1)for d in zip(*chunks)]
return all_dataframes<features_selection>
data_root = '/kaggle/input/titanic'
train_data_path = os.path.join(data_root, 'train.csv')
 test_data_path = os.path.join(data_root, 'test.csv')
 train_data = pd.read_csv(train_data_path)
 test_data = pd.read_csv(test_data_path )
def get_column_indices(df: pd.DataFrame, query_cols: List)-> np.array:

 cols = df.columns.values
sidx = np.argsort(cols)
 return sidx[np.searchsorted(cols, query_cols, sorter=sidx)]<train_model>
use_cols = ['Pclass', 'Sex', 'SibSp', 'Parch']
@time_decorator
def train(model: Union[snapml.RandomForestClassifier, skl.RandomForestClassifier], X_train: np.ndarray, y_train: np.array, weights: np.array):

 model.fit(X_train, y_train, weights )<choose_model_class>
learn_data, valid_data = train_test_split(train_data, test_size=0.3, random_state=0)
 X = pd.get_dummies(learn_data.loc[:, use_cols] ).values
y = learn_data.loc[:, 'Survived'].values
valid_X = pd.get_dummies(valid_data.loc[:, use_cols] ).values
valid_y = valid_data.loc[:, 'Survived'].values
def create_model(library: str, params: Dict, r_seed: int, n_cpus: int)-> Union[snapml.RandomForestClassifier, skl.RandomForestClassifier]:

 if library == 'snapml':
return snapml.RandomForestClassifier(**params, use_gpu=False, random_state=r_seed, n_jobs=n_cpus)
 elif library == 'sklearn':
return skl.RandomForestClassifier(**params, random_state=r_seed, n_jobs=n_cpus )<count_missing_values>
from xgboost import XGBClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
print(f'Number of NaN entries in train: {sum(df.isnull().sum())}, test: {sum(df_test.isnull().sum())}.' )<compute_test_metric>
cls = DecisionTreeClassifier()
cls.fit(X, y)
 y_pred = cls.predict(valid_X)
 y_true = valid_y
print(f1_score(y_true=y_true, y_pred=y_pred))
print(accuracy_score(y_true=y_true, y_pred=y_pred))
corr = np.tril(features.corr() , k=-1)
 print(f'Maximal correlation: {corr.max() :.4f}, minimal correlation {corr.min() :.4f}.' )<compute_train_metric>
test_X = pd.get_dummies(test_data.loc[:, use_cols] ).values
test_y_pred = cls.predict(test_X)
 predict_df = pd.DataFrame()
predict_df['PassengerId'] = test_data['PassengerId']
predict_df['Survived'] = test_y_pred
predict_df.to_csv('submission.csv', index=False )