Kaggle competition: Conway's Reverse Game of Life 2020
<SOS> metric: PostProcessorKernel Kaggle data source: conways-reverse-game-of-life-2020<save_to_csv>
notebook_start = time.perf_counter()
display(HTML())
 %load_ext autoreload
%autoreload 2
test_x = df2.loc[testdf.index]
test_x['Survived'] =(((test_x.WomanOrBoySurvived <= 0.238)&(test_x.Sex > 0.5)&(test_x.Alone > 0.5)) | \ 
(( test_x.WomanOrBoySurvived > 0.238)& \
~(( test_x.WomanOrBoySurvived > 0.55)&(test_x.WomanOrBoySurvived <= 0.633))))
pd.DataFrame({'Survived': test_x['Survived'].astype(int)}, \
index=testdf.index ).reset_index().to_csv('survived.csv', index=False )<define_variables>
! python3 -m pip install -q z3-solver
! apt-get install -qq tree moreutils
LB_simple_rule = 0.83253<train_model>
!rm -rf /ai-games/
!git clone https://github.com/JamesMcGuigan/ai-games/ /ai-games/
!cp -rf /ai-games/puzzles/game_of_life/*./
!rm -rf /kaggle/working/neural_networks/
!cd /ai-games/; git log -n1
model_LR = LinearRegression().fit(train,target.to_numpy().flatten())
 model_Autofeat = LinearRegression().fit(X_train_feature_creation, target.to_numpy().flatten() )<predict_on_test>
from utils.util import *
from utils.plot import *
from utils.game import *
from utils.datasets import *
from utils.tuplize import *
from hashmaps.crop import *
from hashmaps.hash_functions import *
from hashmaps.translation_solver import *
from hashmaps.repeating_patterns import *
from constraint_satisfaction.fix_submission import *
test['Survived_LR'] = np.clip(model_LR.predict(test),0,1)
 test['Survived_AF'] = np.clip(model_Autofeat.predict(X_test_feature_creation),0,1 )<find_best_params>
@njit()
def get_concentric_prime_mask(shape: Tuple[int,int]=(25,25)) -> np.ndarray:
pattern = 'diamond'
assert shape[0] == shape[1]
assert pattern in [ 'diamond', 'oval' ]
x =(shape[0])//2
y =(shape[1])//2
max_r = max(shape)+ 1 if max(shape)% 2 == 0 else max(shape)
 mask = np.zeros(shape, dtype=np.int64)
 for r in range(max_r):
primes = hashable_primes[:r+1]
for dr in range(r+1):
if pattern == 'diamond': prime = primes[r]
elif pattern == 'oval': prime = primes[r] + primes[dr]
coords = { 
(x+(r-dr),y+(dr)) , 
(x-(r-dr),y+(dr)) , 
(x+(r-dr),y-(dr)) , 
(x-(r-dr),y-(dr)) ,
}
for coord in coords:
if min(coord)>= 0 and max(coord)< min(shape):
mask[coord] = prime
return mask
@njit()
def hash_geometric_concentric(board: np.ndarray)-> int:

 assert board.shape[0] == board.shape[1]
mask = get_concentric_prime_mask(shape=board.shape)
 hashed = 0
for x in range(board.shape[0]):
for y in range(board.shape[1]):
for dx in range(mask.shape[0]):
for dy in range(mask.shape[1]):
coords =(( x+dx)%board.shape[0],(y+dy)%board.shape[1])
 hashed += board[coords] * mask[dx,dy]
return hashed
hash_geometric = hash_geometric_concentric
def hyperopt_gb_score(params):
clf = GradientBoostingClassifier(**params)
 current_score = cross_val_score(clf, X_train_feature_creation, target, cv=10 ).mean()
print(current_score, params)
 return current_score
space_gb = {
'n_estimators': hp.choice('n_estimators', range(100, 1000)) ,
'max_depth': hp.choice('max_depth', np.arange(2, 10, dtype=int))
}
best = fmin(fn=hyperopt_gb_score, space=space_gb, algo=tpe.suggest, max_evals=10)
 print('best:')
 print(best )<find_best_params>
def label_board(board):

 tessellation = tessellate_board(board)
 tessellation = scipy.ndimage.convolve(tessellation, [[0,1,0],[1,1,1],[0,1,0]] ).astype(np.bool ).astype(np.int8)
 labeled = skimage.measure.label(tessellation, background=0, connectivity=2)
 labeled = detessellate_board(labeled)
 return labeled
def extract_clusters(board: np.ndarray)-> List[np.ndarray]:
labeled = label_board(board)
 return extract_clusters_from_labels(board, labeled)
 def extract_clusters_from_labels(board: np.ndarray, labeled: np.ndarray)-> List[np.ndarray]:
labels = np.unique(labeled)
 clusters = []
for label in labels:
cluster = board *(labeled == label)
 clusters.append(cluster)
 return clusters
params = space_eval(space_gb, best)
 params<train_model>
dataset_size = 100 if os.environ.get('KAGGLE_KERNEL_RUN_TYPE')== 'Interactive' else 400_000
print(f'dataset_size = {dataset_size}' )
gradient_boosting = GradientBoostingClassifier(**params)
 gradient_boosting.fit(X_train_feature_creation, target)
 Y_pred = gradient_boosting.predict(X_test_feature_creation ).astype(int)
 gradient_boosting.score(X_train_feature_creation, target)
 acc_gradient_boosting = round(gradient_boosting.score(X_train_feature_creation, target)* 100, 2)
 acc_gradient_boosting<save_to_csv>
def read_gzip_pickle_file(filename: str)-> Any:
try:
if not os.path.exists(filename): raise FileNotFoundError
with open(filename, 'rb')as file:
data = file.read()
try: data = gzip.decompress(data)
 except: pass
data = pickle.loads(data)
 except Exception as exception:
data = None
return data
def save_gzip_pickle_file(data: Any, filename: str, verbose=True)-> int:
try:
with open(filename, 'wb')as file:
data = pickle.dumps(data)
 data = gzip.compress(data)
 file.write(data)
 file.close()
filesize = os.path.getsize(filename)
 if verbose: print(f'wrote: {filename} = {humanize.naturalsize(filesize)}')
 return filesize
except:
return 0
pd.DataFrame({'Survived': test['Survived_LR'].astype(int)}, \
index=testdf.index ).reset_index().to_csv('survived_LR16.csv', index=False)
 pd.DataFrame({'Survived': test['Survived_AF'].astype(int)}, \
index=testdf.index ).reset_index().to_csv('survived_Autofeat16.csv', index=False)
 pd.DataFrame({'Survived': Y_pred}, \
index=testdf.index ).reset_index().to_csv('survived_GBC16.csv', index=False )<define_variables>
def generate_cluster_history_lookup(dataset_size=3_000_000//25, forward_play=25, verbose=True):
time_start = time.perf_counter()
csv_size = len(train_df.index)
 dataset = np.concatenate([
csv_to_numpy_list(train_df, key='start'),
generate_random_boards(max(1, dataset_size - csv_size))
])[:dataset_size]
cluster_history_lookup = get_cluster_history_lookup(dataset, forward_play=forward_play)
 time_taken = time.perf_counter() - time_start
if verbose: print(f'{len(cluster_history_lookup)} unique clusters in {time_taken:.1f}s = {1000*time_taken/len(dataset):.0f}ms/board')
 return cluster_history_lookup
cluster_history_lookup_cachefile = f'{output_directory}/cluster_history_lookup.pickle'
cluster_history_lookup = read_gzip_pickle_file(cluster_history_lookup_cachefile)
 if __name__ == '__main__':
cluster_history_lookup = generate_cluster_history_lookup(dataset_size=dataset_size)
 save_gzip_pickle_file(cluster_history_lookup, cluster_history_lookup_cachefile)
LB_LR16 = 0.69377
LB_Autofeat16 = 0.67942
LB_GBC16 = 0.82296<train_model>
def image_segmentation_dataframe_solver(df, history, submission_df=None, exact=False, blank_missing=True, verbose=True):
time_start = time.perf_counter()
stats = { "partial": 0, "exact": 0, "total": 0 }
submission_df = submission_df if submission_df is not None else sample_submission_df.copy()
idxs = df.index
deltas = csv_to_delta_list(df)
 boards = csv_to_numpy_list(df, key='stop')
 labeleds = Parallel(-1 )(delayed(label_board )(board)for board in boards)
 clustereds = Parallel(-1 )(delayed(extract_clusters_from_labels )(board, labels)for board, labels in zip(boards, labeleds))
for idx, delta, stop_board, labels, clusters in zip(idxs, deltas, boards, labeleds, clustereds):
start_board = image_segmentation_solver(
 stop_board, delta, history=history, blank_missing=blank_missing,
labels=labels, clusters=clusters 
)
 is_valid = is_valid_solution(start_board, stop_board, delta)
 if is_valid: stats['exact'] += 1
elif np.count_nonzero(start_board): stats['partial'] += 1
stats['total'] += 1
if is_valid or not exact:
submission_df.loc[idx] = numpy_to_series(start_board, key='start')
 time_taken = time.perf_counter() - time_start
stats['time_seconds'] = int(time_taken)
 stats['time_hours'] = round(time_taken/60/60, 2)
 if verbose: print('image_segmentation_solver() ', stats)
 return submission_df
def image_segmentation_solver(stop_board, delta, history=None, blank_missing=True, labels=None, clusters=None):
history = history if history is not None else cluster_history_lookup
labels = labels if labels is not None else label_board(stop_board)
 clusters = clusters if clusters is not None else extract_clusters_from_labels(stop_board, labels)
 labels = np.unique(labels)
 now_hashes = Parallel(-1 )(delayed(hash_geometric )(cluster)for cluster in clusters)
 new_clusters = {}
for label, now_cluster, now_hash in zip(labels, clusters, now_hashes):
if label == 0: continue
if np.count_nonzero(now_cluster)== 0: continue
if history.get(now_hash,{} ).get(delta,None):
for past_hash in history[now_hash][delta].keys() :
try:
start_cluster = history[now_hash][delta][past_hash]['start']
stop_cluster = history[now_hash][delta][past_hash]['stop']
transform_fn = solve_translation(stop_cluster, now_cluster)
 past_cluster = transform_fn(start_cluster)
 new_clusters[label] = past_cluster
break
except Exception as exception:
pass
if not label in new_clusters:
if blank_missing: new_clusters[label] = np.zeros(now_cluster.shape, dtype=np.int8)
 else: new_clusters[label] = now_cluster
start_board = np.zeros(stop_board.shape, dtype=np.int8)
 for cluster in new_clusters.values() :
start_board += cluster
start_board = start_board.astype(np.bool ).astype(np.int8)
 return start_board
model_LR2 = LinearRegression().fit(train2,target.to_numpy().flatten())
 model_Autofeat2 = LinearRegression().fit(X_train_feature_creation2, target.to_numpy().flatten() )<predict_on_test>
submission_df = image_segmentation_dataframe_solver(test_df[:dataset_size], history=cluster_history_lookup, exact=False)
 submission_df.to_csv('submission.csv' )
test2['Survived_LR'] = np.clip(model_LR2.predict(test2),0,1)
 test2['Survived_AF'] = np.clip(model_Autofeat2.predict(X_test_feature_creation2),0,1 )<find_best_params>
!(for FILE in $(find./.. /input/ -name 'submission.csv' | sort); do cat $FILE | grep ',1' | wc -l | tr '
' ' '; echo $FILE; done)| sort -n;
!find./.. /input/ -name 'submission.csv' | xargs cat | sort -nr | uniq | awk -F',' '!a[$1]++' | sort -n >./submission.csv
!(for FILE in $(find./.. /input/ -name 'submission.csv' | sort); do cat $FILE | grep ',1' | wc -l | tr '
' ' '; echo $FILE; done)| sort -n;
!PYTHONPATH='.' python3./constraint_satisfaction/fix_submission.py