Kaggle competition: Conway's Reverse Game of Life 2020
surnames = df[['surname', 'Name']].groupby('surname' ).count().rename(columns={'Name':'count'})
 df['surname_count'] = df['surname'].apply(lambda x: surnames.loc[x])
 useful.append('surname_count' )<sort_values>
%%writefile main-RT.cpp
extern "C" {
};
using namespace std;
constexpr int n = 25;
constexpr int N = n * n;
using CW = vector<uint8_t>;
using TStamp = std::chrono::time_point<std::chrono::system_clock>;
struct task {
string id;
CW field;
int steps;
};
random_device rd;
mt19937 g(rd());
static atomic<bool> is_stopped;
static atomic<bool> is_timeouted;
double total_timeout = 8.5 * 60 * 60;
TStamp app_started = std::chrono::system_clock::now() ;
void terminate(int code){
is_stopped = true;
is_timeouted = true;
exit(code);
}
vector<string> split(std::string s, char delimiter){
s += delimiter;
vector<string> result;
size_t start = 0;
for(size_t i = start; i < s.size() ; ++i){
while(s[i] != delimiter){
++i;
}
result.push_back(s.substr(start, i - start)) ;
start = i + 1;
}
return result;
}
CW s2vec(const std::string& s){
CW v(N);
if(static_cast<int>(s.size())!= N + N - 1){
cerr << "wtf s2vec format len" << s.size() << endl;
terminate(1);
}
for(int i = 0; i < N; ++i){
v[i] = s[i + i] == '1';
}
return v;
}
string vec2s(const CW& v){
string res;
for(int i = 0; i < N; ++i){
if(v[i]){
res += "1,";
} else {
res += "0,";
}
}
res.resize(res.size() - 1);
return res;
}
int weight(const task& t){
int count = 0;
for(auto k : t.field){
if(k)++count;
}
return count;
}
vector<int> init_nums(int N){
vector<int> values;
for(int i = 0; i < N; ++i){
values.push_back(i);
}
return values;
}
const vector<int> n_nums = init_nums(N);
bool calc(const CW& field, int pos, int x, int y)noexcept {
static const int dx8[] = {-1, -1, -1, 0, 0, 1, 1, 1};
static const int dy8[] = {-1, 0, 1, -1, 1, -1, 0, 1};
int count = 0;
for(int i = 0; i < 8; ++i){
int a = x + dx8[i];
int b = y + dy8[i];
if(a < 0)a += n;
else if(a >= n)a -= n;
if(b < 0)b += n;
else if(b >= n)b -= n;
if(field[b * n + a]){
++count;
}
}
return(count | field[pos])== 3;
}
bool calc_left_line(const CW& field, size_t pos)noexcept {
static const size_t d_left8[] = { -1ULL, -25ULL, -24ULL, +24, +1, +25 + 24, +25, +26};
uint8_t count = 0;
for(int i = 0; i < 8; ++i){
count += field[pos + d_left8[i]];
}
return(count | field[pos])== 3;
}
bool calc_right_line(const CW& field, size_t pos)noexcept {
static const size_t d_right8[] = { -26ULL, -25ULL, - 49ULL, -1ULL, -24ULL, +24, +25, +1};
uint8_t count = 0;
for(int i = 0; i < 8; ++i){
count += field[pos + d_right8[i]];
}
return(count | field[pos])== 3;
}
bool calc_bottom_line(const CW& field, size_t pos)noexcept {
static const size_t d_bottom8[] = { -26ULL, -25ULL, -24ULL, -1ULL, 1, -599ULL, -600ULL, -601ULL};
uint8_t count = 0;
for(int i = 0; i < 8; ++i){
count += field[pos + d_bottom8[i]];
}
return(count | field[pos])== 3;
}
bool calc_top_line(const CW& field, size_t pos)noexcept {
static const size_t d_top8[] = { +599, +600, +601, -1ULL, 1, +24, +25, +26};
uint8_t count = 0;
for(int i = 0; i < 8; ++i){
count += field[pos + d_top8[i]];
}
return(count | field[pos])== 3;
}
bool calc_no_checks(const CW& field, size_t pos)noexcept {
static const size_t di8[] = {0, 1, 2, 25, 27, +50, +51, +52};
uint8_t count = 0;
for(int i = 0; i < 8; ++i){
count += field[pos + di8[i]];
}
return(count | field[pos + 26])== 3;
}
CW run_step(CW start)noexcept {
thread_local CW next(N);
for(int x = 1; x < n - 1; ++x){
next[x] = calc_top_line(start, x);
next[600 + x] = calc_bottom_line(start, 600 + x);
next[x * 25] = calc_left_line(start, x * 25);
next[x * 25 + 24] = calc_right_line(start, x * 25 + 24);
}
size_t pos = 0;
for(int y = 1; y < n - 1; ++y){
for(int x = 1; x < n - 1; ++x){
next[pos + 26] = calc_no_checks(start, pos);
++pos;
}
pos += 2;
}
for(auto i : {0, n-1, N - n, N - 1}){
next[i] = calc(start, i, i % n, i / n);
}
return next;
}
void run_step_partial(CW* steps, int x, int y, int step, vector<pair<int, int>>& changelog){
int range = step + 1;
for(int b = y - range; b <= y + range; ++b){
int q = b;
if(q < 0)q += n;
else if(q >= n)q -= n;
for(int a = x - range; a <= x + range; ++a){
int p = a;
if(p < 0)p += n;
else if(p >= n)p -= n;
size_t i = q * n + p;
bool not_vert_side = p > 0 && p + 1 < n;
bool not_hori_side = q > 0 && q + 1 < n;
bool value;
if(not_vert_side && not_hori_side){
value = calc_no_checks(steps[step], i - 26);
} else {
bool not_vert_side = p > 0 && p + 1 < n;
bool not_hori_side = q > 0 && q + 1 < n;
if(not_vert_side && q + 1 == n){
value = calc_bottom_line(steps[step], i);
} else if(not_vert_side && q == 0){
value = calc_top_line(steps[step], i);
} else if(not_hori_side && p == 0){
value = calc_left_line(steps[step], i);
} else if(not_hori_side && p + 1 == n){
value = calc_right_line(steps[step], i);
} else {
value = calc(steps[step], i, p, q);
}
}
bool previous = steps[step + 1][i];
if(value != previous){
changelog.push_back({step + 1, i});
steps[step + 1][i] = value;
}
}
}
}
int run_step_partial_last(CW* steps, int x, int y, int step, vector<pair<int, int>>& changelog, const CW& original){
int range = step + 1;
int diff = 0;
for(int b = y - range; b <= y + range; ++b){
int q = b;
if(q < 0)q += n;
else if(q >= n)q -= n;
for(int a = x - range; a <= x + range; ++a){
int p = a;
if(p < 0)p += n;
else if(p >= n)p -= n;
size_t i = q * n + p;
bool not_vert_side = p > 0 && p + 1 < n;
bool not_hori_side = q > 0 && q + 1 < n;
bool value;
if(not_vert_side && not_hori_side){
value = calc_no_checks(steps[step], i - 26);
} else {
if(not_vert_side && q + 1 == n){
value = calc_bottom_line(steps[step], i);
} else if(not_vert_side && q == 0){
value = calc_top_line(steps[step], i);
} else if(not_hori_side && p == 0){
value = calc_left_line(steps[step], i);
} else if(not_hori_side && p + 1 == n){
value = calc_right_line(steps[step], i);
} else {
value = calc(steps[step], i, p, q);
}
}
bool previous = steps[step + 1][i];
if(value != previous){
if(value == original[i]){
++diff;
} else {
--diff;
}
changelog.push_back({step + 1, i});
steps[step + 1][i] = value;
}
}
}
return diff;
}
void revert_changes(CW* steps, const vector<pair<int, int>>& changelog){
for(auto i : changelog){
steps[i.first][i.second] = !steps[i.first][i.second];
}
}
void print(const CW& field){
for(int pos = 0; pos < N; ++pos){
cout <<(field[pos] ? '*' : '.'); //static_cast<int>(field[pos]);
if(pos % n == n - 1){
cout << endl;
}
}
cout << endl;
}
int validate(CW start, const task& t){
for(int i = 0; i < t.steps; ++i){
start = run_step(start);
}
int count = 0;
for(int i = 0; i < N; ++i){
if(start[i] == t.field[i])
 ++count;
}
return count;
}
int validate_partial(const task& t, CW* steps, int x, int y, vector<pair<int, int>>& changelog){
changelog.clear() ;
for(int i = 0; i < t.steps - 1; ++i){
run_step_partial(steps, x, y, i, changelog);
}
int diff = run_step_partial_last(steps, x, y, t.steps - 1, changelog, t.field);
return diff;
}
static class Cache : private boost::noncopyable {
private:
map<string, pair<int, string>> cache;
mutex m;
public:
Cache() { }
auto get(const std::string& id){
lock_guard<mutex> g(m);
return cache[id];
}
bool set(const pair<int, CW>& value, const task& t){
const std::string& id = t.id;
{
lock_guard<mutex> g(m);
if(cache.find(id)!= cache.end() && cache[id].first >= value.first){
return false;
}
}
int score = validate(value.second, t);
if(score != value.first){
cerr << "wtf score for " << id << ": " << score << ' ' << value.first << endl;
for(auto i : value.second){
cerr <<(int)i << ' ';
}
cerr << endl;
throw std::runtime_error("wtf score");
}
string s = vec2s(value.second);
{
lock_guard<mutex> g(m);
cache[id] = {value.first, s};
ofstream cache_file("processed/cache-real-time", std::ofstream::app);
cache_file << id << ' ' << value.first << ' ' << s << endl;
return true;
}
}
} global_cache;
CW prepare_mask(CW solution, const task& t){
for(int i = 0; i < t.steps; ++i){
solution = run_step(solution);
}
vector<int> diff;
for(int i = 0; i < N; ++i){
if(t.field[i] != solution[i]){
diff.push_back(i);
}
}
for(int i = 0; i < t.steps; ++i){
vector<int> next_diff;
for(auto pos : diff){
next_diff.push_back(i);
int x = pos % n;
int y = pos / n;
static const int dx8[] = {-1, -1, -1, 0, 0, 1, 1, 1};
static const int dy8[] = {-1, 0, 1, -1, 1, -1, 0, 1};
for(int i = 0; i < 8; ++i){
int a = x + dx8[i];
int b = y + dy8[i];
if(a < 0)a += n;
else if(a >= n)a -= n;
if(b < 0)b += n;
else if(b >= n)b -= n;
next_diff.push_back(b * n + a);
}
}
diff = next_diff;
sort(diff.begin() , diff.end());
{
auto last = std::unique(diff.begin() , diff.end());
diff.erase(last, diff.end());
}
}
CW result(N);
for(auto i : diff){
result[i] = true;
}
// cerr << diff.size() << " for " << global_cache.get(t.id ).first << endl;
return result;
}
CW shake_single_if_equal(CW _field, const task& t, bool fast = false){
CW steps[t.steps + 1];
steps[0] = _field;
CW& field = steps[0];
vector<pair<int, int>> changelog;
int best_score = validate(field, t);
auto best_pos = field;
int score = best_score;
int current_score = best_score;
for(int i = 0; i < t.steps; ++i){
steps[i + 1] = run_step(steps[i]);
}
CW mask_to_shake = prepare_mask(best_pos, t);
int default_tries = fast ? 39 : 99;
int default_range = fast ? 2 : 3;
int tries = default_tries;
int range = default_range;
do {
if(score != best_score){
tries = default_tries;
score = best_score;
range = default_range;
}
if(tries % 50 == 0)range -= 1;
int shift = rand() % N;
bool dir = rand() % 2;
int i = shift;
int x = i % n;
int y = i / n;
for(int index = 0; index < N; ++index){
if(dir){
++i;
++x;
if(x == n){
x = 0;
++y;
if(y == n)y = 0;
}
if(i == N)i = 0;
} else {
--i;
--x;
if(x == -1){
x = n - 1;
--y;
if(y == -1)y = n - 1;
}
if(i == N)i = 0;
if(i < 0)i = N - 1;
}
if(!mask_to_shake[i] && tries % 5 != 0)continue; /// _---------- RT_ADJ
bool value = field[i];
field[i] = !field[i];
int new_score = current_score + validate_partial(t, steps, x, y, changelog);
if(new_score > best_score){
best_score = new_score;
best_pos = field;
global_cache.set({best_score, best_pos}, t);
} else if(new_score < best_score - range){
field[i] = !field[i];
} else {
if(rand() % 4 == 0){
field[i] = !field[i];
}
}
if(value == field[i]){
revert_changes(steps, changelog);
} else {
current_score = new_score;
}
}
} while(best_score != score || tries-- > 0);
return best_pos;
}
CW shake_k_and_single(const task &t, CW start, int tries, bool fast = false){
auto next_start = start;
vector<int> indexes = n_nums;
shuffle(indexes.begin() , indexes.end() , g);
for(int i = 0; i < tries; ++i){
next_start[indexes[i]] = !next_start[indexes[i]];
}
next_start = shake_single_if_equal(next_start, t, true);
return shake_single_if_equal(next_start, t, fast);
}
CW make_random() {
CW c(N);
auto indexes = n_nums;
shuffle(indexes.begin() , indexes.end() , g);
indexes.resize(N / 2);
for(auto i : indexes){
c[i] = 1;
}
return c;
}
CW make_checked() {
CW c(N);
for(size_t i = 0; i < N; i += 2){
c[i] = true;
}
return c;
}
namespace CNF_solver {
using CNF = std::vector<std::vector<int>>;
bool alive(const CW& field, int pos, int x, int y){
static const int dx8[] = {-1, -1, -1, 0, 0, 1, 1, 1};
static const int dy8[] = {-1, 0, 1, -1, 1, -1, 0, 1};
int count = 0;
for(int i = 0; i < 8; ++i){
int a = x + dx8[i];
int b = y + dy8[i];
if(a < 0)a += n;
else if(a >= n)a -= n;
if(b < 0)b += n;
else if(b >= n)b -= n;
if(field[b * n + a]){
++count;
}
}
return(count | field[pos])== 3;
}
vector<int> generate_positions(int x, int y){
static const int dx8[] = {-1, -1, -1, 0, 0, 1, 1, 1};
static const int dy8[] = {-1, 0, 1, -1, 1, -1, 0, 1};
vector<int> result;
result.push_back(-9999); // place holder
for(int i = 0; i < 8; ++i){
int a = x + dx8[i];
int b = y + dy8[i];
if(a < 0)a += n;
else if(a >= n)a -= n;
if(b < 0)b += n;
else if(b >= n)b -= n;
result.push_back(b * n + a);
}
return result;
}
// CNF: all values false: through negative
bool geti(uint32_t value, int index){
return(value &(1 << index)) ;
}
CNF universal_cnf() {
CNF result_cnf;
uint32_t mask = 0;
while(!geti(mask, 10)) {
int count = 0;
for(int i = 0; i < 8; ++i){
if(geti(mask, i)) {
++count;
}
}
bool alive =(count | geti(mask, 8)) == 3;
bool is_true =(alive == geti(mask, 9)) ;
if(!is_true){
vector<int> next;
for(int i = 0; i <= 9; ++i){
if(geti(mask, i)) {
next.push_back(-i - 1);
} else {
next.push_back(i + 1);
}
}
result_cnf.push_back(next);
}
sort(result_cnf.begin() , result_cnf.end());
++mask;
}
// 325065 CNF -> 160k CNF
// speed increased 1.6 ; 30 ->
// optimize_cnf(result_cnf);
// Optimized much better externally by python tool sympy:
// $ python3
// >>>
// >>> x, y, z = symbols('x y z')
 // >>> simplify_logic(( x & y & ~z)|(x & ~y & z), form="cnf")
 // x &(y | z)&(~y | ~z)
 result_cnf = {{1, 2, 3, 4, 5, 6, 7, -10}, {1, 2, 3, 4, 5, 6, 8, -10}, {1, 2, 3, 4, 5, 6, 9, -10}, {1, 2, 3, 4, 5, 7, 8, -10}, {1, 2, 3, 4, 5, 7, 9, -10}, {1, 2, 3, 4, 5, 8, 9, -10}, {1, 2, 3, 4, 6, 7, 8, -10}, {1, 2, 3, 4, 6, 7, 9, -10}, {1, 2, 3, 4, 6, 8, 9, -10}, {1, 2, 3, 4, 7, 8, 9, -10}, {1, 2, 3, 5, 6, 7, 8, -10}, {1, 2, 3, 5, 6, 7, 9, -10}, {1, 2, 3, 5, 6, 8, 9, -10}, {1, 2, 3, 5, 7, 8, 9, -10}, {1, 2, 3, 6, 7, 8, 9, -10}, {1, 2, 4, 5, 6, 7, 8, -10}, {1, 2, 4, 5, 6, 7, 9, -10}, {1, 2, 4, 5, 6, 8, 9, -10}, {1, 2, 4, 5, 7, 8, 9, -10}, {1, 2, 4, 6, 7, 8, 9, -10}, {1, 2, 5, 6, 7, 8, 9, -10}, {1, 3, 4, 5, 6, 7, 8, -10}, {1, 3, 4, 5, 6, 7, 9, -10}, {1, 3, 4, 5, 6, 8, 9, -10}, {1, 3, 4, 5, 7, 8, 9, -10}, {1, 3, 4, 6, 7, 8, 9, -10}, {1, 3, 5, 6, 7, 8, 9, -10}, {1, 4, 5, 6, 7, 8, 9, -10}, {2, 3, 4, 5, 6, 7, 8, -10}, {2, 3, 4, 5, 6, 7, 9, -10}, {2, 3, 4, 5, 6, 8, 9, -10}, {2, 3, 4, 5, 7, 8, 9, -10}, {2, 3, 4, 6, 7, 8, 9, -10}, {2, 3, 5, 6, 7, 8, 9, -10}, {2, 4, 5, 6, 7, 8, 9, -10}, {3, 4, 5, 6, 7, 8, 9, -10}, {-1, -10, -2, -3, -4}, {-1, -10, -2, -3, -5}, {-1, -10, -2, -3, -6}, {-1, -10, -2, -3, -7}, {-1, -10, -2, -3, -8}, {-1, -10, -2, -4, -5}, {-1, -10, -2, -4, -6}, {-1, -10, -2, -4, -7}, {-1, -10, -2, -4, -8}, {-1, -10, -2, -5, -6}, {-1, -10, -2, -5, -7}, {-1, -10, -2, -5, -8}, {-1, -10, -2, -6, -7}, {-1, -10, -2, -6, -8}, {-1, -10, -2, -7, -8}, {-1, -10, -3, -4, -5}, {-1, -10, -3, -4, -6}, {-1, -10, -3, -4, -7}, {-1, -10, -3, -4, -8}, {-1, -10, -3, -5, -6}, {-1, -10, -3, -5, -7}, {-1, -10, -3, -5, -8}, {-1, -10, -3, -6, -7}, {-1, -10, -3, -6, -8}, {-1, -10, -3, -7, -8}, {-1, -10, -4, -5, -6}, {-1, -10, -4, -5, -7}, {-1, -10, -4, -5, -8}, {-1, -10, -4, -6, -7}, {-1, -10, -4, -6, -8}, {-1, -10, -4, -7, -8}, {-1, -10, -5, -6, -7}, {-1, -10, -5, -6, -8}, {-1, -10, -5, -7, -8}, {-1, -10, -6, -7, -8}, {-10, -2, -3, -4, -5}, {-10, -2, -3, -4, -6}, {-10, -2, -3, -4, -7}, {-10, -2, -3, -4, -8}, {-10, -2, -3, -5, -6}, {-10, -2, -3, -5, -7}, {-10, -2, -3, -5, -8}, {-10, -2, -3, -6, -7}, {-10, -2, -3, -6, -8}, {-10, -2, -3, -7, -8}, {-10, -2, -4, -5, -6}, {-10, -2, -4, -5, -7}, {-10, -2, -4, -5, -8}, {-10, -2, -4, -6, -7}, {-10, -2, -4, -6, -8}, {-10, -2, -4, -7, -8}, {-10, -2, -5, -6, -7}, {-10, -2, -5, -6, -8}, {-10, -2, -5, -7, -8}, {-10, -2, -6, -7, -8}, {-10, -3, -4, -5, -6}, {-10, -3, -4, -5, -7}, {-10, -3, -4, -5, -8}, {-10, -3, -4, -6, -7}, {-10, -3, -4, -6, -8}, {-10, -3, -4, -7, -8}, {-10, -3, -5, -6, -7}, {-10, -3, -5, -6, -8}, {-10, -3, -5, -7, -8}, {-10, -3, -6, -7, -8}, {-10, -4, -5, -6, -7}, {-10, -4, -5, -6, -8}, {-10, -4, -5, -7, -8}, {-10, -4, -6, -7, -8}, {-10, -5, -6, -7, -8}, {1, 10, 2, 3, 4, 5, -6, -7, -8}, {1, 10, 2, 3, 4, 6, -5, -7, -8}, {1, 10, 2, 3, 4, 7, -5, -6, -8}, {1, 10, 2, 3, 4, 8, -5, -6, -7}, {1, 10, 2, 3, 5, 6, -4, -7, -8}, {1, 10, 2, 3, 5, 6, -4, -7, -9}, {1, 10, 2, 3, 5, 6, -7, -8, -9}, {1, 10, 2, 3, 5, 7, -4, -6, -8}, {1, 10, 2, 3, 5, 7, -4, -6, -9}, {1, 10, 2, 3, 5, 7, -4, -8, -9}, {1, 10, 2, 3, 5, 7, -6, -8, -9}, {1, 10, 2, 3, 5, 8, -4, -6, -7}, {1, 10, 2, 3, 5, 8, -6, -7, -9}, {1, 10, 2, 3, 6, 7, -4, -5, -8}, {1, 10, 2, 3, 6, 7, -4, -5, -9}, {1, 10, 2, 3, 6, 7, -5, -8, -9}, {1, 10, 2, 3, 6, 8, -4, -5, -7}, {1, 10, 2, 3, 6, 8, -5, -7, -9}, {1, 10, 2, 3, 7, 8, -4, -5, -6}, {1, 10, 2, 3, 7, 8, -5, -6, -9}, {1, 10, 2, 4, 5, 6, -3, -7, -8}, {1, 10, 2, 4, 5, 7, -3, -6, -8}, {1, 10, 2, 4, 5, 8, -3, -6, -7}, {1, 10, 2, 4, 6, 7, -3, -5, -8}, {1, 10, 2, 4, 6, 8, -3, -5, -7}, {1, 10, 2, 4, 7, 8, -3, -5, -6}, {1, 10, 2, 5, 6, 7, -3, -4, -8}, {1, 10, 2, 5, 6, 7, -3, -4, -9}, {1, 10, 2, 5, 6, 7, -3, -8, -9}, {1, 10, 2, 5, 6, 8, -3, -4, -7}, {1, 10, 2, 5, 6, 8, -3, -7, -9}, {1, 10, 2, 5, 7, 8, -3, -4, -6}, {1, 10, 2, 5, 7, 8, -3, -6, -9}, {1, 10, 2, 6, 7, 8, -3, -4, -5}, {1, 10, 2, 6, 7, 8, -3, -5, -9}, {1, 10, 3, 4, 5, 6, -2, -7, -8}, {1, 10, 3, 4, 5, 7, -2, -6, -8}, {1, 10, 3, 4, 5, 8, -2, -6, -7}, {1, 10, 3, 4, 6, 7, -2, -5, -8}, {1, 10, 3, 4, 6, 8, -2, -5, -7}, {1, 10, 3, 4, 7, 8, -2, -5, -6}, {1, 10, 3, 5, 6, 7, -2, -4, -8}, {1, 10, 3, 5, 6, 7, -2, -4, -9}, {1, 10, 3, 5, 6, 7, -2, -8, -9}, {1, 10, 3, 5, 6, 8, -2, -4, -7}, {1, 10, 3, 5, 6, 8, -2, -7, -9}, {1, 10, 3, 5, 7, 8, -2, -4, -6}, {1, 10, 3, 5, 7, 8, -2, -6, -9}, {1, 10, 3, 6, 7, 8, -2, -4, -5}, {1, 10, 3, 6, 7, 8, -2, -5, -9}, {1, 10, 4, 5, 6, 7, -2, -3, -8}, {1, 10, 4, 5, 6, 8, -2, -3, -7}, {1, 10, 4, 5, 7, 8, -2, -3, -6}, {1, 10, 4, 6, 7, 8, -2, -3, -5}, {1, 10, 5, 6, 7, 8, -2, -3, -4}, {1, 10, 5, 6, 7, 8, -2, -3, -9}, {10, 2, 3, 4, 5, 6, -1, -7, -8}, {10, 2, 3, 4, 5, 7, -1, -6, -8}, {10, 2, 3, 4, 5, 8, -1, -6, -7}, {10, 2, 3, 4, 6, 7, -1, -5, -8}, {10, 2, 3, 4, 6, 8, -1, -5, -7}, {10, 2, 3, 4, 7, 8, -1, -5, -6}, {10, 2, 3, 5, 6, 7, -1, -4, -8}, {10, 2, 3, 5, 6, 7, -1, -4, -9}, {10, 2, 3, 5, 6, 7, -1, -8, -9}, {10, 2, 3, 5, 6, 8, -1, -4, -7}, {10, 2, 3, 5, 6, 8, -1, -7, -9}, {10, 2, 3, 5, 7, 8, -1, -4, -6}, {10, 2, 3, 5, 7, 8, -1, -6, -9}, {10, 2, 3, 6, 7, 8, -1, -4, -5}, {10, 2, 3, 6, 7, 8, -1, -5, -9}, {10, 2, 4, 5, 6, 7, -1, -3, -8}, {10, 2, 4, 5, 6, 8, -1, -3, -7}, {10, 2, 4, 5, 7, 8, -1, -3, -6}, {10, 2, 4, 6, 7, 8, -1, -3, -5}, {10, 2, 5, 6, 7, 8, -1, -3, -4}, {10, 2, 5, 6, 7, 8, -1, -3, -9}, {10, 3, 4, 5, 6, 7, -1, -2, -8}, {10, 3, 4, 5, 6, 8, -1, -2, -7}, {10, 3, 4, 5, 7, 8, -1, -2, -6}, {10, 3, 4, 6, 7, 8, -1, -2, -5}, {10, 3, 5, 6, 7, 8, -1, -2, -4}, {10, 3, 5, 6, 7, 8, -1, -2, -9}, {10, 4, 5, 6, 7, 8, -1, -2, -3}};
return result_cnf;
}
// 0...N-1: previous position
// N...2N - 1: current position
vector<std::vector<int>> cnf_universal() {
// universal "conways rules" CNF
CNF cnf;
CNF cnf_conway_cell = universal_cnf() ;
for(int y = 0; y < n; ++y){
for(int x = 0; x < n; ++x){
int pos_previous = y * n + x;
int pos_current = y * n + x + N;
vector<int> positions = generate_positions(x, y);
positions.push_back(pos_previous);
positions.push_back(pos_current);
for(auto& c : cnf_conway_cell){
vector<int> next;
for(auto& p : c){
if(p < 0){
next.push_back(-positions[-p] - 1);
} else {
next.push_back(positions[p] + 1);
}
}
cnf.push_back(next);
}
}
}
cout << "CNF size: " << cnf.size() << endl;
return cnf;
}
const static CNF cnf_conway_rules = cnf_universal() ;
CNF get_cnf(const task& t, const std::pair<int, int>& ignore_range = {0, 0}){
int k = t.steps;
if(k == 1){
int k = t.steps;
CNF cnf = cnf_conway_rules;
for(int iter = 1; iter < k; ++iter){
CNF cnf_i = cnf_conway_rules;
for(auto& i : cnf_i){
for(auto& j : i){
if(j > 0)j += N * iter;
else j -= N * iter;
}
cnf.push_back(i);
}
}
// fulfill CNF for current field
for(int i = 0; i < N; ++i){
int var = i + k * N + 1;
if(t.field[i]){
cnf.push_back({var});
} else {
cnf.push_back({-var});
}
}
return cnf;
}
// First level
CNF cnf = cnf_conway_rules;
// Middle levels: not first, not last
for(int iter = 1; iter + 1 < k; ++iter){
CNF cnf_i = cnf_conway_rules;
for(auto& i : cnf_i){
for(auto& j : i){
if(j > 0)j += N * iter;
else j -= N * iter;
}
cnf.push_back(i);
}
}
// fulfill CNF for current field
CNF cnf_last = cnf_conway_rules;
for(auto& i : cnf_last){
int to_ignore = 0;
for(auto& j : i){
for(int l = N + 1; l <= N + N; ++l){
if(j == l || j == -l){
to_ignore = j;
}
}
}
int pos = -1;
if(to_ignore > 0){
pos = to_ignore - N - 1;
if(t.field[pos])continue;
}
if(to_ignore < 0){
pos = -to_ignore - N - 1;
if(!t.field[-to_ignore - N - 1])continue;
}
if(pos >= ignore_range.first && pos < ignore_range.second){
continue;
}
std::vector<int> c;
for(auto& j : i){
if(j == to_ignore)continue;
if(j > 0)
 c.push_back(j + N *(k - 1)) ;
else
c.push_back(j - N *(k - 1)) ;
}
cnf.push_back(c);
}
return cnf;
}
} // namespace CNF_solver
struct KissatSolver {
KissatSolver(double timeout = 2.1)
 : timeout(timeout)
 , cleaner([this]() -> void {
cleanup() ;
})
 {
}
~KissatSolver() {
cleaner.join() ;
}
kissat* add(string id){
kissat* solver = kissat_init() ;
{
lock_guard<mutex> g(m);
if(all.find(id)!= all.end())throw std::runtime_error("Kissat solver already contains" + id);
all[id] = {solver, std::chrono::system_clock::now() };
}
return solver;
}
void notify_done(string id){
lock_guard<mutex> g(m);
to_free.insert(id);
}
void cleanup() {
while(!is_stopped){
this_thread::sleep_for(300ms);
lock_guard<mutex> g(m);
int to_clean = 0;
for(const auto& id : to_free){
auto& item = all[id];
kissat* solver = item.first;
kissat_release(solver);
all.erase(id);
++to_clean;
}
to_free.clear() ;
int to_terminate = 0;
TStamp now = std::chrono::system_clock::now() ;
for(const auto& i : all){
auto item = i.second;
double running = std::chrono::duration_cast<std::chrono::seconds>(now - item.second ).count() ;
if(running > timeout){
kissat_terminate(item.first);
++to_terminate;
}
}
if(to_terminate || to_clean){
cout << "terminated " << to_terminate << " cleaned " << to_clean << endl;
}
}
}
private:
double timeout;
map<string, std::pair<kissat*, TStamp>> all;
thread cleaner;
set<string> to_free;
mutex m;
} kissat_solver;
void sat_solver_k(const task& t){
int k = t.steps;
CNF_solver::CNF cnf = CNF_solver::get_cnf(t);
kissat* solver = kissat_solver.add(t.id);
kissat_set_option(solver, "quiet", 1);
kissat_set_configuration(solver, "sat");
kissat_reserve(solver, N * k);
for(auto c : cnf){
for(auto i : c){
kissat_add(solver, i);
}
kissat_add(solver, 0);
}
int result = kissat_solve(solver);
switch(result){
case 20:
cerr << "wtf solver failed for " << t.id << endl;
break;
}
CW solution;
bool at_least_one_true = 0;
for(int i = 1; i <= N; ++i){
int next = kissat_value(solver, i);
if(next > 0){
solution.push_back(1);
at_least_one_true = true;
} else {
solution.push_back(0);
}
}
kissat_solver.notify_done(t.id);
int score = validate(solution, t);
global_cache.set({score, solution}, t);
if(score != N && at_least_one_true){
shake_single_if_equal(solution, t);
}
}
bool solve_sat(const task& t){
auto it = global_cache.get(t.id);
if(is_timeouted)return false;
if(it.first == N){
return true;
}
if(t.steps == 1){
sat_solver_k(t);
} else if(t.steps == 2 && weight(t)< 100){ /// ------------ RT_ADJ
sat_solver_k(t);
}
return global_cache.get(t.id ).first == N;
}
bool solve_fast(const task& t){
if(is_timeouted)return false;
auto it = global_cache.get(t.id);
if(it.first == N){
return true;
}
{
if(weight(t)< 200){
shake_single_if_equal(CW(N), t, true);
} else {
// shake_single_if_equal(make_random() , t, true);
}
shake_single_if_equal(make_checked() , t, true);
if(t.steps % 2 == 0)
 shake_single_if_equal(t.field, t, true);
else
shake_single_if_equal(run_step(t.field), t, true);
}
// enabled this only as efficient
int score = it.first;
size_t iters = 5;
CW next = s2vec(it.second);
while(iters--){
CW next2 = shake_single_if_equal(next, t, true);
if(validate(next2, t)>= validate(next, t)) {
next = next2;
}
it = global_cache.get(t.id);
int new_score = it.first;
if(new_score == N)break;
if(new_score != score){
score = new_score;
next = s2vec(it.second);
iters = 11;
}
}
return global_cache.get(t.id ).first == N;
}
bool solve_slower(const task& t){
if(is_timeouted)return false;
auto it = global_cache.get(t.id);
if(it.first == N){
return true;
}
// enabled this only as efficient
int score = it.first;
size_t iters = 4;
CW next = s2vec(it.second);
while(iters--){
CW next2 = shake_single_if_equal(next, t);
CW next3 = shake_k_and_single(t, next2, 8);
if(validate(next3, t)>= validate(next, t)) {
next = next3;
}
if(validate(next2, t)>= validate(next, t)) {
next = next2;
}
it = global_cache.get(t.id);
int new_score = it.first;
if(new_score == N)break;
if(new_score != score){
score = new_score;
next = s2vec(it.second);
iters = 6;
}
}
/*
for(int i = 0; i < 4; ++i){
CW next = make_random() ;
for(int j = 0; j < 5; ++j){
next = shake_k_and_single(t, next, 30);
}
}
*/
// trying to run upfront far
// run_upfront(id, end, steps);
return global_cache.get(t.id ).first == N;
}
void validate_training(const std::string& file){
ifstream input(file);
string line;
input >> line;
while(input >> line){
vector<string> v = split(line, ',');
if(v.size() != static_cast<size_t>(N + N + 2)) {
cerr << "incorrect input within line within validate: " << line << endl;
terminate(1);
}
string id = v[0];
int steps = stoi(v[1]);
CW start(N);
CW end(N);
for(int i = 0; i < N; ++i){
start[i] =(v[i + 2] == "1");
end[i] =(v[N + i + 2] == "1");
}
if(validate(start, {id, end, steps})!= N){
cerr << "validate failed at: " << id << endl;
terminate(1);
}
}
}
pair<int, int> calc_stats(const std::vector<task>& tasks){
int count = 0;
int count_solved = 0;
int total_score = 0;
for(const task & t : tasks){
++count;
int score = global_cache.get(t.id ).first;
total_score += score;
if(score == N)++count_solved;
}
int max_score = count * N;
cout << "Solved: " << count_solved << "/" << count ;
cout << " :<>: " << total_score << "/" << max_score << " = " << static_cast<double>(total_score)/ max_score << endl;;
return {count_solved, total_score};
}
void print_stats(const vector<task>& tasks){
auto started = std::chrono::system_clock::now() ;
auto first_stats = calc_stats(tasks);
while(!is_stopped){
sleep(9);
{ // time-out for application ?
TStamp now = std::chrono::system_clock::now() ;
double total_running = std::chrono::duration_cast<std::chrono::seconds>(now - app_started ).count() ;
if(total_running > total_timeout){
is_timeouted = true;
}
}
double running =(std::chrono::system_clock::now() - started ).count() ;
auto current = calc_stats(tasks);
double speed1 =(current.first - first_stats.first)/ running * 1e9;
double speed2 =(current.second - first_stats.second)/ running * 1e9;
cout << "speed " << std::fixed << std::setprecision(6)<< speed1 << ' ' << speed2 << ' ';
}
calc_stats(tasks);
}
int main(int argc, char *argv[]){
validate_training("train.csv");
if(argc > 1 && strcmp(argv[1], "--only-validate")== 0){
cerr << "Validated train!" << endl;
terminate(0);
}
if(argc > 1 && strcmp(argv[1], "--timeout")== 0){
total_timeout = stoi(argv[2]);
}
ifstream input("test.csv");
string line;
// header
input >> line;
std::vector<task> all_tasks;
while(input >> line){
vector<string> v = split(line, ',');
if(v.size() != static_cast<size_t>(N + 2)) {
cerr << "incorrect input within line: " << line << endl;
terminate(1);
}
string id = v[0];
int steps = stoi(v[1]);
// if(id >= "51000")continue;
// if(steps != 3)continue;
CW field(N);
for(int i = 0; i < N; ++i){
field[i] = v[i + 2] == "1";
}
task t{id, field, steps};
CW empty(N);
global_cache.set({validate(empty, t), empty}, t);
all_tasks.push_back(t);
}
if(argc > 1 && strcmp(argv[1], "--shuffle")== 0){
cout << "shuffle tasks randomly" << endl;
shuffle(all_tasks.begin() , all_tasks.end() , g);
}
if(argc > 1 && strcmp(argv[1], "--sort-score")== 0){
cout << "sort by score" << endl;
sort(all_tasks.begin() , all_tasks.end() , [](const auto& i, const auto& j){
int score_a = global_cache.get(i.id ).first;
int score_b = global_cache.get(j.id ).first;
return score_a < score_b;
});
}
if(argc > 1 && strcmp(argv[1], "--sort-score-reverse")== 0){
cout << "sort by score(reverse" << endl;
sort(all_tasks.begin() , all_tasks.end() , [](const auto& i, const auto& j){
int score_a = global_cache.get(i.id ).first;
int score_b = global_cache.get(j.id ).first;
return score_a > score_b;
});
}
if(argc > 1 && strcmp(argv[1], "--sort-weight")== 0){
cout << "sort by weight" << endl;
sort(all_tasks.begin() , all_tasks.end() , [](const auto& i, const auto& j){
return(weight(i)< weight(j)) ;
});
}
cout << "solve SAT" << endl;
{
boost::asio::thread_pool pool(std::thread::hardware_concurrency());
for(const auto& t : all_tasks){
if(global_cache.get(t.id ).first == N)continue;
boost::asio::post(pool, [t]() {
solve_sat(t);
} 
);
}
thread stats([all_tasks]() {print_stats(all_tasks); });
pool.join() ;
is_stopped = true;
stats.join() ;
}
cout << "solve fast" << endl;
{
is_stopped = false;
boost::asio::thread_pool pool(std::thread::hardware_concurrency());
for(const auto& t : all_tasks){
if(global_cache.get(t.id ).first == N)continue;
boost::asio::post(pool, [t]() {
solve_fast(t);
} 
);
}
thread stats([all_tasks]() {print_stats(all_tasks); });
pool.join() ;
is_stopped = true;
stats.join() ;
}
while(!is_timeouted){
cout << "solve slow" << endl;
sort(all_tasks.begin() , all_tasks.end() , [](const auto& i, const auto& j){
return(weight(i)> weight(j)) ;
});
is_stopped = false;
boost::asio::thread_pool pool(std::thread::hardware_concurrency());
for(const auto& t : all_tasks){
if(global_cache.get(t.id ).first == N)continue;
boost::asio::post(pool, [t]() {
solve_slower(t);
} 
);
}
thread stats([all_tasks]() {print_stats(all_tasks); });
pool.join() ;
is_stopped = true;
stats.join() ;
}
}
df['title'] = df['Name'].apply(lambda n: n.split(',')[1].split('.')[0].strip())
 df[['title', 'Survived']].groupby('title' ).agg({'Survived': [
'mean', 'std', 'count']} ).sort_values(( 'Survived', 'count'), ascending=False )<categorify>
!apt-get install libboost-all-dev -y
title_groups = {
"Capt": "sacrifies",
"Col": "army",
"Rev": "sacrifies",
"Major": "army",
"Mr" : "Mr",
"Master": "Master",
"Miss": "Miss",
"Mrs": "Mrs",
"Mme": "Mrs",
"Ms": "Mrs",
"Mlle": "Miss"
}
df['title_group'] = df['title'].apply(lambda t: title_groups[t] if t in title_groups else 'other')
 df = pd.concat([df, pd.get_dummies(df['title_group'], prefix='title_group')], axis=1)
 t_g = df['title_group'].unique()
useful.extend(['title_group_{}'.format(x)for x in t_g] )<sort_values>
if KAGGLE:
pass
else:
!g++ -o main-RT main-RT.cpp -std=c++17 -g -O3 -lpthread -Wall -Wextra -march=native -funroll-loops -fno-omit-frame-pointer kissat/build/libkissat.a kissat/build/handle.o
<prepare_x_and_y><EOS> 
%%writefile dump.cpp
using namespace std;
int N = 625;
int main() {
map<string, pair<int, string>> cache;
{
ifstream cache_file("processed/cache");
string id;
int score;
string field;
while(cache_file >> id >> score >> field){
auto it = cache[id];
if(it.first < score){
cache[id] = {score, field};
}
}
}
string time_suffix = to_string(time(nullptr)) ;
ofstream dump_file("processed/dump-" + time_suffix + ".csv");
ofstream shrink_cache("processed/shrink-cache-" + time_suffix);
dump_file << "id";
for(int i = 0; i < N; ++i){
dump_file << ",start_" << i;
}
dump_file << '
';
for(auto i : cache){
dump_file << i.first << "," << i.second.second << '
';
shrink_cache << i.first << ' ' << i.second.first << ' ' << i.second.second << '
';
}
}