Kaggle competition: Forest Cover Type Prediction
path = np.array(pd.read_csv('.. /input/close-ends-chunks-optimization-aka-2-opt/1516404.851601243.csv' ).Path )<randomize_order>
train_data=pd.read_csv('/kaggle/input/forest-cover-type-prediction/train.csv')
 train_data.head()
def not_trivial_permutations(iterable):
perms = permutations(iterable)
 next(perms)
 yield from perms
@lru_cache(maxsize=None)
 def not_trivial_indexes_permutations(length):
return np.array([list(p)for p in not_trivial_permutations(range(length)) ])
 path_index = np.argsort(path[:-1])
 print(f'Total score is {score_path(path):.2f}.')
 for _ in range(3):
for ids in tqdm(triplets):
i, j, k = sorted(path_index[ids])
 head, tail = path[i-1], path[k+1]
chunks = [path[i:i+1], path[i+1:j], path[j:j+1], path[j+1:k], path[k:k+1]]
chunks = [chunk for chunk in chunks if len(chunk)]
scores = [chunk_scores(chunk)for chunk in chunks]
default_score = score_compound_chunk(i-1, head, chunks, tail, scores)
 best_score = default_score
for indexes_permutation in not_trivial_indexes_permutations(len(chunks)) :
score = score_compound_chunk(i-1, head, chunks, tail, scores, indexes_permutation)
 if score < best_score:
permutation = [chunks[i] for i in indexes_permutation]
best_chunk = np.concatenate([[head], np.concatenate(permutation), [tail]])
 best_score = score
if best_score < default_score:
path[i-1:k+2] = best_chunk
path_index = np.argsort(path[:-1])
 print(f'New total score is {score_path(path):.2f}.Permutating path at indexes {i}, {j}, {k}.')
 triplets = triplets[:10**6]<compute_test_metric>
test_data=pd.read_csv('/kaggle/input/forest-cover-type-prediction/test.csv')
 test_data.head()
print(f'Final score is {score_path(path):.2f}.' )<save_to_csv>
train_data['Cover_Type'].value_counts()
def make_submission(name, path):
pd.DataFrame({'Path': path} ).to_csv(f'{name}.csv', index=False )<compute_test_metric>
from sklearn.model_selection import train_test_split
make_submission(score_path(path), path )<import_modules>
from sklearn.model_selection import train_test_split
import numpy as np
import pandas as pd
import time
import math
import numba
import sympy
from sympy.utilities.iterables import multiset_permutations<data_type_conversions>
X=train_data.drop(labels=['Id','Cover_Type'],axis=1)
 y=train_data['Cover_Type']
def load_cities(filename):
cities = pd.read_csv(filename)
 city_id = cities.CityId.astype(np.int32)
 loc = np.vstack([cities.X.astype(np.float32), cities.Y.astype(np.float32)] ).transpose()
is_prime = np.array([1 if sympy.isprime(i)else 0 for i in city_id], dtype=np.int32)
 return(city_id, loc, is_prime)
 def load_tour(filename):
tour = pd.read_csv(filename)
 tour = tour.Path.values.astype(np.int32)
 return tour<save_to_csv>
X_train,X_val,y_train,y_val=train_test_split(X,y,random_state=12 )
def save_tour(filename, tour):
with open(filename, "w")as f:
f.write("Path
")
 for i in tour:
f.write(str(i))
f.write("
" )<compute_test_metric>
from sklearn.ensemble import RandomForestClassifier
@numba.jit('f4(f4[:], f4[:])', nopython=True)
 def euc_2d(a, b):
xd = a[0] - b[0]
yd = a[1] - b[1]
return math.sqrt(xd * xd + yd * yd)
 @numba.jit('f8(i4[:], f4[:,:], i4[:])', nopython=True, parallel=True)
 def cost_santa2018(tour, loc, is_prime):
dist = 0.0
for i in numba.prange(1, tour.shape[0]):
a = tour[i - 1]
b = tour[i]
d = euc_2d(loc[a], loc[b])
 if i % 10 == 0 and is_prime[a] == 0:
d *= 1.1
dist += d
return dist<statistical_test>
rfc=RandomForestClassifier(n_estimators=70)
 rfc.fit(X_train,y_train )
def gen_subprobs(n, k):
return np.array([[i, i + k] for i in range(n - k)], dtype=np.int32)
 def gen_perms(k):
idx = np.arange(1, k - 1, dtype=np.int32)
 perm = []
for p in multiset_permutations(idx):
perm.append(np.hstack([[0], p, [k-1]]))
return np.array(perm, dtype=np.int32)
 @numba.jit('f4(i4[:], f4[:,:], i4[:], i4[:])', nopython=True)
 def subprob_cost(tour, dist_mat, s_10th, s_prime):
dist = 0.0
for t in range(1, tour.shape[0]):
i = tour[t - 1]
j = tour[t]
d = dist_mat[i][j]
if s_10th[t] != 0 and s_prime[i] != 0:
d *= 1.1
dist += d
return dist
@numba.jit('i4(f4[:], i4[:,:,:], i8, i4[:,:], i4[:], f4[:,:], i4[:], i4[:,:])', nopython=True, parallel=True)
 def finetune_(best_improvement, best_perm, k, subprobs, tour, loc, is_prime, perm):
updated = 0
init_tour = np.arange(k ).astype(np.int32)
 for t in numba.prange(subprobs.shape[0]):
idx = np.arange(subprobs[t][0], subprobs[t][1] ).astype(np.int32)
 s_loc = loc[tour[idx]]
s_prime =(1 - is_prime[tour[idx]] ).astype(np.int32)
 s_10th = np.empty(( k,), dtype=np.int32)
 s_dist_mat = np.empty(( k, k), dtype=np.float32)
 for i in range(k):
s_10th[i] = 1 if(i + idx[0])% 10 == 0 else 0
for i in range(k):
for j in range(i + 1, k):
s_dist_mat[i][j] = s_dist_mat[j][i] = euc_2d(s_loc[i], s_loc[j])
 costs = np.empty(( perm.shape[0],), dtype=np.float32)
 init_cost = subprob_cost(init_tour, s_dist_mat, s_10th, s_prime)
 for i in range(perm.shape[0]):
costs[i] = subprob_cost(perm[i], s_dist_mat, s_10th, s_prime)
 best_index = np.argmin(costs)
 delta = init_cost - costs[best_index]
if delta > best_improvement[t]:
best_perm[t][0] = idx
best_perm[t][1] = idx[perm[best_index]]
best_improvement[t] = delta
updated += 1
return updated
def finetune(k, subprobs, tour, loc, is_prime, perm):
updated = 0
best_perm = np.empty(( subprobs.shape[0], 2, k), dtype=np.int32)
 best_improvement = np.zeros(( subprobs.shape[0],), dtype=np.float32)
 finetune_(best_improvement, best_perm, k, subprobs, tour, loc, is_prime, perm)
 used = []
updates = np.argsort(best_improvement)[::-1]
for i in updates:
delta = best_improvement[i]
if delta > 0:
s = best_perm[i][0][0]
e = best_perm[i][0][-1]
if any([(r[0] < s and s < r[1])or(r[0] < e and e < r[1])for r in used]):
continue
used.append(( s, e))
tour[best_perm[i][0]] = tour[best_perm[i][1]]
updated += 1
else:
break
return updated<load_from_csv>
rfc.score(X_val,y_val )
city_id, loc, is_prime = load_cities(".. /input/traveling-santa-2018-prime-paths/cities.csv")
 tour = load_tour(".. /input/dataset/submission.csv")
 init_cost = cost_santa2018(tour, loc, is_prime)
 print("initial cost", init_cost )<find_best_params>
predict=rfc.predict(test_data.drop(labels=['Id'],axis=1))
K = 10 + 2
subprobs = gen_subprobs(city_id.shape[0], K)
 subprob_perms = gen_perms(K)
 print("** perm={}!".format(K-2))
t = time.time()
updated = finetune(K, subprobs, tour, loc, is_prime, subprob_perms)
 cost = cost_santa2018(tour, loc, is_prime)
 print("cost: {:.4f}, improvement: {:.4f}, found: {}, time: {:.2f}".format(cost, init_cost - cost, updated, time.time() - t))
save_tour("submission.csv", tour )<feature_engineering>
Submission=pd.DataFrame(data=predict,columns=['Cover_Type'])
 Submission.head()
def score_tour(tour):
df = cities.reindex(tour + [0] ).reset_index()
df['prime'] = df.CityId.isin(primes ).astype(int)
 df['dist'] = np.hypot(df.X - df.X.shift(-1), df.Y - df.Y.shift(-1))
df['penalty'] = df['dist'][9::10] *(1 - df['prime'][9::10])* 0.1
return df.dist.sum() + df.penalty.sum()
cities = pd.read_csv('.. /input/traveling-santa-2018-prime-paths/cities.csv')
 primes = list(primerange(0, len(cities['CityId'].values)))
 primes += [0,1]
cities['NotPrime'] = np.abs(cities.CityId.isin(primes ).astype(int)- 1)* 0.1 + 1
dPaths = {v:{'X': x, 'Y': y, 'penalty': z} for v, x, y, z in cities[['CityId','X','Y','NotPrime']].values}
cities = pd.read_csv('.. /input/traveling-santa-2018-prime-paths/cities.csv', index_col=['CityId'])
 tour = pd.read_csv('.. /input/dp-shuffle/submission.csv')['Path'].tolist()
score_tour(tour )<statistical_test>
Submission['Id']=test_data['Id']
Submission.set_index('Id',inplace=True )
<save_to_csv><EOS> 
Submission.to_csv('Submission.csv' )