Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<drop_column>
data = data.drop(columns=[
'PoolQC',
'Fence',
'MiscFeature',
'Alley',
'FireplaceQu'
] )<data_type_conversions>
int_outputs = ".. /input/intermediate-outputs"
with open(int_outputs+"/test_matrices_all.p", "rb")as filehandle:
test_matrices = pickle.load(filehandle )
for col in data:
datatype = data[col].dtype
if datatype == 'object':
data[col] = data[col].fillna('None' )<drop_column>
def get_counts2(column, trees,stem = True):
t1 = time.clock()
stemmer = None
if stem: stemmer = nl.stem.snowball.SnowballStemmer("english")
 counts = []
for i in column.index:
tree = trees[i]
if tree:
counts_i = get_rels_ch3(tree,stemmer)
 counts.append(counts_i)
 else:
counts.append(None)
 t2 = time.clock()
print(i,":",t2 - t1)
 return counts
labels = train_data.pop('SalePrice')
 data = data.drop(columns=['SalePrice'] )<categorify>
def get_probs(train_dict,test_list,ids,weights,smoothing=0.5):
t0=time.clock()
train_cats = {i:v.to_dense().fillna(0)for(i, v)in train_dict.items() if i != "TOTAL"}
train_total = train_dict["TOTAL"].to_dense().fillna(0)
 all_rows = train_total.index
all_columns = train_total.columns
results = pd.DataFrame(index=ids.index, columns=['id']+[i for i in train_cats],dtype=float)
 for(n,test_mat)in enumerate(test_list):
cat_probs = pd.Series({i:1.0 for i in train_cats})
 if test_mat is not None:
test_matrix = test_mat.to_dense().fillna(0)
 nonzeros = test_matrix[test_matrix>0].stack().stack()
for(j, count)in nonzeros.iteritems() :
row=(j[0],j[1] if(j[0],j[1])in all_rows else "_OTHER")
 column =(j[3] if(j[3],j[2])in all_columns else "_OTHER", j[2])
 column_total_nonzeros = train_total[column][train_total[column] >0].size
if column_total_nonzeros == 0:
column =("_OTHER", j[2])
 column_total_nonzeros = train_total[column][train_total[column] >0].size
for(cat,train_matrix)in train_cats.items() :
numerator = smoothing
denominator =(column_total_nonzeros+1)*smoothing
if(column in train_matrix.columns and column_total_nonzeros >=2):
denominator = denominator + train_matrix.loc[:,column].sum()
if(row in train_matrix.index):
numerator = numerator +train_matrix.loc[row,column]
cat_probs[cat] = cat_probs[cat]*pow(numerator / denominator,count)
 cat_probs = cat_probs/cat_probs.sum()
cat_probs=cat_probs * weights
results.loc[ids.index[n]] = cat_probs/cat_probs.sum()
print(n,":",time.clock() - t0)
 results['id'] = ids
return results
features = pd.get_dummies(data )<split>
def probs_regress(train_dict,test_list,ids,weights,smoothing=0.5, regressors={}):
probs = get_probs(train_dict,test_list,ids,weights,smoothing)
 num_columns = probs.columns.drop('id')
 for i, v in regressors.items() :
probs[i] = regressors[i].predict(probs[i])
 results_sum = probs[num_columns].sum(axis=1)
 probs[num_columns] = probs[num_columns].divide(results_sum, axis=0)
 return probs
train_features = features[:train_data.shape[0]]
test_features = features[train_data.shape[0]:]<train_model>
train_data = pd.read_csv('.. /input/spooky-author-identification/train.csv')
 test_data = pd.read_csv('.. /input/spooky-author-identification/test.csv')
 int_outputs = ".. /input/intermediate-outputs"
with open(int_outputs+"/test_matrices_all.p", "rb")as filehandle:
test_matrices_all = pickle.load(filehandle)
 with open(int_outputs+"/train_matrices_master.p", "rb")as filehandle:
train_matrices_master = pickle.load(filehandle)
 with open(int_outputs+"/train_matrices_calib.p", "rb")as filehandle:
train_matrices_calib = pickle.load(filehandle)
 with open(int_outputs+"/train_matrices_holdout.p", "rb")as filehandle:
train_matrices_holdout = pickle.load(filehandle)
 with open(int_outputs+"/isotonic_regs.p", "rb")as filehandle:
isotonic_regs = pickle.load(filehandle)
 probs = probs_regress(train_dict=train_matrices_master,
test_list = test_matrices_all,
ids = test_data['id'],
weights = train_data['author'][:16000].value_counts() ,
smoothing=0.5,
regressors=isotonic_regs 
)
 probs.to_csv('submission.csv',index=False )
<compute_train_metric><EOS> 
def prep2(train_dict, threshold):
train_cats = {i:v.to_dense().fillna(0)for(i, v)in train_dict.items() if i != "TOTAL"}
train_total = train_dict["TOTAL"].to_dense().fillna(0)
 row_sums=train_total.sum(axis=1)
 all_oth_rows = train_total.loc[row_sums < threshold,:].drop([("LEAF", "_OTHER"),("NODE", "_OTHER")],axis=0)
 all_oth_leaf = all_oth_rows.loc['LEAF']
all_oth_node = all_oth_rows.loc['NODE']
all_oth_leaf_ind = pd.MultiIndex.from_product([["LEAF"],all_oth_leaf.index])
 all_oth_node_ind = pd.MultiIndex.from_product([["NODE"],all_oth_node.index])
 train_total.loc[('LEAF', '_OTHER'),:] = all_oth_leaf.sum(axis=0)
 train_total.loc[('NODE', '_OTHER'),:] = all_oth_node.sum(axis=0)
 train_total = train_total.drop(all_oth_leaf_ind, axis=0)
 train_total = train_total.drop(all_oth_node_ind, axis=0)
 all_oth_l_ind = train_total.columns.intersection(pd.MultiIndex.from_product([all_oth_node.index,["L"]]))
all_oth_r_ind = train_total.columns.intersection(pd.MultiIndex.from_product([all_oth_node.index,["R"]]))
all_oth_c_ind = train_total.columns.intersection(pd.MultiIndex.from_product([all_oth_node.index,["C"]]))
all_oth_l = train_total.loc[:,all_oth_l_ind]
all_oth_r = train_total.loc[:,all_oth_r_ind]
all_oth_c = train_total.loc[:,all_oth_c_ind]
train_total.loc[:,('_OTHER', 'L')] = all_oth_l.sum(axis=1)
 train_total.loc[:,('_OTHER', 'R')] = all_oth_r.sum(axis=1)
 train_total.loc[:,('_OTHER', 'C')] = all_oth_c.sum(axis=1)
 train_total = train_total.drop(all_oth_l_ind, axis=1)
 train_total = train_total.drop(all_oth_r_ind, axis=1)
 train_total = train_total.drop(all_oth_c_ind, axis=1)
 for(i, matrix)in train_cats.items() :
oth_leaf_ind = matrix.index.intersection(all_oth_leaf_ind)
 oth_node_ind = matrix.index.intersection(all_oth_node_ind)
 oth_leaf = matrix.loc[oth_leaf_ind,:]
oth_node = matrix.loc[oth_node_ind,:]
train_cats[i].loc[('LEAF', '_OTHER'),:] = oth_leaf.sum(axis=0)
 train_cats[i].loc[('NODE', '_OTHER'),:] = oth_node.sum(axis=0)
 train_cats[i] = train_cats[i].drop(oth_leaf_ind, axis=0)
 train_cats[i] = train_cats[i].drop(oth_node_ind, axis=0)
 oth_l_ind = matrix.columns.intersection(all_oth_l_ind)
 oth_r_ind = matrix.columns.intersection(all_oth_r_ind)
 oth_c_ind = matrix.columns.intersection(all_oth_c_ind)
 oth_l = matrix.loc[:, oth_l_ind]
oth_r = matrix.loc[:, oth_r_ind]
oth_c = matrix.loc[:, oth_c_ind]
train_cats[i].loc[:,('_OTHER', 'L')] = oth_l.sum(axis=1)
 train_cats[i].loc[:,('_OTHER', 'R')] = oth_r.sum(axis=1)
 train_cats[i].loc[:,('_OTHER', 'C')] = oth_c.sum(axis=1)
 train_cats[i] = train_cats[i].drop(columns=oth_l_ind)
 train_cats[i] = train_cats[i].drop(columns=oth_r_ind)
 train_cats[i] = train_cats[i].drop(columns=oth_c_ind)
 return {'TOTAL':train_total, **train_cats}