Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<groupby>
from collections import ChainMap
from multiprocessing import cpu_count
from pathlib import Path
cluster_mean = train_scaled_df[["km_cluster","SalePrice","TotalSF","OverallQual","Age","Total_Bathrooms","YearRemodAdd","GarageArea",
"MSZoning","OverallCond","KitchenQual","FireplaceQu"]].groupby("km_cluster" ).mean().reset_index()<feature_engineering>
import numpy as np
import pandas as pd
import lightgbm as lgb
from sklearn.externals.joblib import Parallel, delayed
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split, StratifiedKFold
from tqdm import tqdm_notebook as tqdm
from tsfresh.feature_extraction.feature_calculators import *
from tsfresh.feature_selection.relevance import calculate_relevance_table
cluster_mean = cluster_mean.T<import_modules>
seed = 1
np.random.seed(seed )
from sklearn.decomposition import PCA<train_on_grid>
ROOT = Path.cwd().parent/'input'
SAMPLE = ROOT/'sample_submission.csv'
TRAIN = ROOT/'X_train.csv'
TARGET = ROOT/'y_train.csv'
TEST = ROOT/'X_test.csv'
ID_COLS = ['series_id', 'measurement_number']
x_cols = {
'series_id': np.uint32,
'measurement_number': np.uint32,
'orientation_X': np.float32,
'orientation_Y': np.float32,
'orientation_Z': np.float32,
'orientation_W': np.float32,
'angular_velocity_X': np.float32,
'angular_velocity_Y': np.float32,
'angular_velocity_Z': np.float32,
'linear_acceleration_X': np.float32,
'linear_acceleration_Y': np.float32,
'linear_acceleration_Z': np.float32
}
y_cols = {
'series_id': np.uint32,
'group_id': np.uint32,
'surface': str
}
pca = PCA(n_components=2)
 house_pca = pca.fit(train_scaled ).transform(train_scaled )<create_dataframe>
x_trn = pd.read_csv(TRAIN, usecols=x_cols.keys() , dtype=x_cols)
 x_tst = pd.read_csv(TEST, usecols=x_cols.keys() , dtype=x_cols)
 y_trn = pd.read_csv(TARGET, usecols=y_cols.keys() , dtype=y_cols )
house_pca_df = pd.DataFrame(house_pca)
 house_pca_df.columns = ["pca1","pca2"]<concatenate>
def part(f, **params):

 def wrapper(x): return f(x, **params)
 wrapper.__name__ = f.__name__
return wrapper
train_scaled_df = pd.concat([train_scaled_df, house_pca_df], axis=1 )<set_options>
class SliceFeatures:

 def __init__(self, mode='first', n=5):
if mode not in {'first', 'middle', 'last'}:
raise ValueError('unexpected mode')
 self.mode = mode
self.n = n
def __call__(self, data):
if self.mode == 'first':
start, end = 0, self.n
elif self.mode == 'last':
start, end = -self.n, len(data)
 elif self.mode == 'middle':
mid = len(data)// 2
div, mod = divmod(self.n, 2)
 start, end = mid-div, mid+div+mod
cols = data.columns
vec = data.iloc[start:end].values.T.ravel()
new_cols = [
f'{col}_{self.mode}{i}'
for i in range(self.n)for col in cols]
return dict(zip(new_cols, vec))
my_colors = plt.rcParams['axes.prop_cycle'].by_key() ['color']<create_dataframe>
def generate_features(data, features, ignore=None):

 with Parallel(n_jobs=cpu_count())as parallel:
extracted = parallel(delayed(generate_features_for_group )(
 group=group.drop(columns=ignore or []),
features=features 
)for _, group in tqdm(data.groupby('series_id')))
 return pd.DataFrame(extracted )
pca_comp_df = pd.DataFrame(pca.components_,columns=train_scaled_df.drop(["km_cluster","pca1","pca2"],axis=1 ).columns ).T
pca_comp_df.columns = ["pca1","pca2"]<feature_engineering>
def generate_features_for_group(group, features):

 return dict(ChainMap(*[feat(group)for feat in features]))
train_df_le.loc[train_df["SalePrice"] >= 278000, "high_class"] = 1<data_type_conversions>
funcs =(
 mean, median, standard_deviation, variance,
skewness, kurtosis, maximum, minimum,
mean_change, mean_abs_change, count_above_mean, count_below_mean,
mean_second_derivative_central, sum_of_reoccurring_data_points,
abs_energy, sum_values, sample_entropy,
longest_strike_above_mean, longest_strike_below_mean,
first_location_of_minimum, first_location_of_maximum,
*[part(large_standard_deviation, r=r*0.05)for r in range(1, 20)],
*[part(autocorrelation, lag=lag)for lag in range(1, 25)],
*[part(number_peaks, n=n)for n in(1, 2, 3, 5, 7, 10, 25, 50)],
*[part(c3, lag=lag)for lag in range(1, 5)],
*[part(quantile, q=q)for q in (.1,.2,.3,.4,.5,.6,.7,.8,.9)],
part(partial_autocorrelation, param=[
{'lag': lag} for lag in range(25)
 ]),
part(agg_autocorrelation, param=[
{'f_agg': s, 'maxlag': 40} for s in('mean', 'median', 'var')
 ]),
part(linear_trend, param=[
{'attr': a} for a in 
('pvalue', 'rvalue', 'intercept', 'slope', 'stderr')
 ])
 )
train_df_le["high_class"] = train_df_le["high_class"].fillna(0 )<install_modules>
features = [
StatsFeatures(funcs),
SliceFeatures('first'),
SliceFeatures('middle'),
SliceFeatures('last')
 ]
!pip install pydotplus<import_modules>
ignore = ['series_id', 'measurement_number']
from sklearn import tree
import pydotplus
from six import StringIO<prepare_x_and_y>
print('Feature extraction on train dataset')
 x_trn = generate_features(x_trn, features, ignore=ignore )
tree_x = train_df_le[["TotalSF","OverallQual","Age","GrLivArea","GarageCars","Total_Bathrooms","GarageType",
"YearRemodAdd","GarageArea","CentralAir","MSZoning","OverallCond","KitchenQual","FireplaceQu","1stFlrSF"]]
tree_y = train_df_le[["high_class"]]<train_model>
print('Feature extraction on test dataset')
 x_tst = generate_features(x_tst, features, ignore=ignore )
clf = tree.DecisionTreeClassifier(max_depth=4)
 clf = clf.fit(tree_x, tree_y )<save_to_csv>
enc = LabelEncoder()
y_trn = pd.Series(enc.fit_transform(y_trn['surface']))
dot_data = StringIO()
tree.export_graphviz(clf, out_file=dot_data,feature_names=tree_x.columns)
 graph = pydotplus.graph_from_dot_data(dot_data.getvalue() )<set_options>
def accuracy(y_true, y_pred):
n = len(y_true)
 y_hat = y_pred.reshape(9, n ).argmax(axis=0)
 value =(y_true == y_hat ).mean()
return 'accuracy', value, True
Image(graph.create_png() )<install_modules>
model = lgb.LGBMClassifier(n_estimators=3000, learning_rate=0.005,
colsample_bytree=0.4, objective='multiclass',
num_leaves=500, num_class=9 )
!pip install -q git+https://github.com/tensorflow/docs iteration-utilities<import_modules>
x_train, x_valid, y_train, y_valid = train_test_split(x_trn, y_trn, test_size=0.1, random_state=seed )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
pd.set_option('display.max_columns', None)
 pp = pprint.PrettyPrinter()<load_from_csv>
model.fit(x_train, y_train,
eval_set=[(x_valid, y_valid)],
eval_metric=accuracy,
early_stopping_rounds=300,
verbose=150 )
def load_dataset(path: str)-> pd.DataFrame:
df = pd.read_csv(path)
 return df
train_csv = load_dataset("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test_csv = load_dataset("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 train_csv.sample(10 )<define_variables>
test = enc.inverse_transform(model.predict(x_tst))
<filter><EOS> 
submit = pd.read_csv(SAMPLE)
 submit['surface'] = test
submit.to_csv('submit.csv', index=None )