Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<randomize_order>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
def random_choice_prob_index(a, axis=1):
r = np.expand_dims(np.random.rand(a.shape[1-axis]), axis=axis)
 return(a.cumsum(axis=axis)> r ).argmax(axis=axis )<randomize_order>
import nltk
import string
from nltk.corpus import stopwords
from nltk.stem.snowball import SnowballStemmer
import re
def lucky_choice_search(top_k, fam_size, original, choice_matrix,
disable_tqdm=False, n_iter=100000000,
verbose=10000, random_state=2019):
best = original.copy()
best_score = cost_function(np.int32(best))
if random_state is not None:
np.random.seed(random_state)
 fam_indices = np.random.choice(range(choice_matrix.shape[0]), size=fam_size, p=fam_weight)
 for i in tqdm(range(n_iter), disable=disable_tqdm):
new = best.copy()
new[fam_indices] = choice_matrix[fam_indices, random_choice_prob_index(choice_weight[fam_indices])]
new_score = cost_function(np.int32(new))
if new_score < best_score:
best_score = new_score
best = new
print(f'{i} NEW BEST SCORE: ', best_score)
 submission['assigned_day'] = best
submission.to_csv(f'submission_{best_score}.csv')
 if verbose and i % verbose == 0:
print(f"Iteration
return best, best_score<choose_model_class>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
best, best_score = lucky_choice_search(
 choice_matrix=choice_matrix,
top_k=top_k,
fam_size=20,
original=original,
n_iter=250000000,
disable_tqdm=False,
random_state=20191217,
verbose=None
)<save_to_csv>
def clean_text(text):
text = text.translate(string.punctuation)
 text = text.lower().split()
stops = set(stopwords.words("english"))
text = [w for w in text if not w in stops and len(w)>= 3]
text = " ".join(text)
 text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
 text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r",", " ", text)
 text = re.sub(r"\.", " ", text)
 text = re.sub(r"!", " ! ", text)
 text = re.sub(r"\/", " ", text)
 text = re.sub(r"\^", " ^ ", text)
 text = re.sub(r"\+", " + ", text)
 text = re.sub(r"\-", " - ", text)
 text = re.sub(r"\=", " = ", text)
 text = re.sub(r"'", " ", text)
 text = re.sub(r"(\d+ )(k)", r"\g<1>000", text)
 text = re.sub(r":", " : ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r" u s ", " american ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e - mail", "email", text)
 text = re.sub(r"j k", "jk", text)
 text = re.sub(r"\s{2,}", " ", text)
 text = text.split()
stemmer = SnowballStemmer('english')
 stemmed_words = [stemmer.stem(word)for word in text]
text = " ".join(stemmed_words)
 return text
print("Clean Text Function defined" )
submission['assigned_day'] = best
submission.to_csv(f'submission_{best_score}.csv' )<import_modules>
print("Start Cleaning Data texts")
 train_df["question_text"] = train_df["question_text"].map(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].map(lambda x: clean_text(x))
print("Data Questions Text Cleaned" )
import numpy as np
import pandas as pd
from numba import njit, prange<load_from_csv>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
data = pd.read_csv('/kaggle/input/santa-workshop-tour-2019/family_data.csv', index_col='family_id')
 family_size = data.n_people.values.astype(np.int8)
 penalties = np.asarray([
[
0,
50,
50 + 9 * n,
100 + 9 * n,
200 + 9 * n,
200 + 18 * n,
300 + 18 * n,
300 + 36 * n,
400 + 36 * n,
500 + 36 * n + 199 * n,
500 + 36 * n + 398 * n
] for n in range(family_size.max() + 1)
 ])
 family_cost_matrix = np.concatenate(data.n_people.apply(lambda n: np.repeat(penalties[n, 10], 100 ).reshape(1, 100)))
 for fam in data.index:
for choice_order, day in enumerate(data.loc[fam].drop("n_people")) :
family_cost_matrix[fam, day - 1] = penalties[data.loc[fam, "n_people"], choice_order]
accounting_cost_matrix = np.zeros(( 500, 500))
for n in range(accounting_cost_matrix.shape[0]):
for diff in range(accounting_cost_matrix.shape[1]):
accounting_cost_matrix[n, diff] = max(0,(n - 125.0)/ 400.0 * n**(0.5 + diff / 50.0))
@njit(fastmath=True)
 def cost_function(prediction, family_size, family_cost_matrix, accounting_cost_matrix):
N_DAYS = family_cost_matrix.shape[1]
MAX_OCCUPANCY = 300
MIN_OCCUPANCY = 125
penalty = 0
daily_occupancy = np.zeros(N_DAYS + 1, dtype=np.int16)
 for i,(pred, n)in enumerate(zip(prediction, family_size)) :
daily_occupancy[pred - 1] += n
penalty += family_cost_matrix[i, pred - 1]
accounting_cost = 0
n_low = 0
n_high = 0
daily_occupancy[-1] = daily_occupancy[-2]
for day in range(N_DAYS):
n_next = daily_occupancy[day + 1]
n = daily_occupancy[day]
n_high +=(n > MAX_OCCUPANCY)
 n_low +=(n < MIN_OCCUPANCY)
 diff = abs(n - n_next)
 accounting_cost += accounting_cost_matrix[n, diff]
return np.asarray([penalty, accounting_cost, n_low, n_high])
 def score(prediction):
fc, ac, l, h = cost_function(prediction, family_size, family_cost_matrix, accounting_cost_matrix)
 return(fc + ac)+(l + h)* 1000000
fam = pd.read_csv("/kaggle/input/santa-workshop-tour-2019/family_data.csv")
 pref = fam.values[:,1:-1]<load_from_csv>
print("Creating Model by GLOVE Embeddings")
 EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
pred = pd.read_csv('/kaggle/input/santa-ip/submission.csv', index_col='family_id' ).assigned_day.values
init_score = score(pred)
 print(init_score )<load_from_csv>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
sub = pd.read_csv('/kaggle/input/santa-ip/submission.csv')
 !cp /kaggle/input/santa-ip/submission.csv./submission_72398.91780918743.csv<compute_test_metric>
print("Starting Prediction by GLOVE Embeddings for VAL")
 pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))
print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))
if score > best_score:
best_thresh = thresh
best_score = score
def seed_finding(seed, prediction_input):
prediction = prediction_input.copy()
np.random.seed(seed)
 best_score = score(prediction)
 original_score = best_score
print("SEED: {} ORIGINAL SCORE: {}".format(seed, original_score))
for t in range(100):
for i in range(5000):
for j in range(10):
di = prediction[i]
prediction[i] = pref[i, j]
cur_score = score(prediction)
 KT = 1
if t < 5:
KT = 1.5
elif t < 10:
KT = 4.5
else:
if cur_score > best_score + 100:
KT = 3
elif cur_score > best_score + 50 :
KT = 2.75
elif cur_score > best_score + 20:
KT = 2.5
elif cur_score > best_score + 10:
KT = 2
elif cur_score > best_score:
KT = 1.5
else:
KT = 1
prob = np.exp(-(cur_score - best_score)/ KT)
 if np.random.rand() < prob:
best_score = cur_score
else:
prediction[i] = di
if best_score < original_score:
print("NEW BEST SCORE on seed {}: {}".format(seed, best_score))
sub.assigned_day = prediction
sub.to_csv(f'submission_{best_score}.csv', index=False)
 break
if best_score >= original_score:
print("UNLUCKY on seed {} for 100 runs, no impovement.".format(seed))
return prediction, best_score<find_best_params>
print()
print("Best Tresh for GLOVE is {0} at score {1}".format(best_thresh,best_score))
print()
pred_glove_val_y =(pred_glove_val_y>best_thresh )
best_score = init_score
for seed in range(1201, 1225):
pred, best_score = seed_finding(seed, pred)
 if best_score < init_score:
init_score = best_score
else:
best_score = init_score
pred = pd.read_csv(f'submission_{best_score}.csv', index_col='family_id' ).assigned_day.values<load_from_csv>
print("Prediction for test based on GLOVE Embeddings")
 pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_glove_test_y =(pred_glove_test_y>best_thresh)
 print("Glove Emebddings Prediction Complete" )
train=pd.read_csv('/kaggle/input/titanic/train.csv')
 test=pd.read_csv('/kaggle/input/titanic/test.csv')
 submission=pd.DataFrame(test['PassengerId'])
 y=train['Survived']<concatenate>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
dataset=pd.concat([train.drop(['PassengerId','Survived'],axis=1),test.drop('PassengerId',axis=1)] )<drop_column>
print("Creating Model by Wikinews Embeddings")
 EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
dataset['Age'].fillna(dataset['Age'].mean() ,inplace=True)
 dataset['Fare'].fillna(dataset['Fare'].median() ,inplace=True)
 dataset['Embarked'].fillna('S',inplace=True)
 dataset.drop(['Cabin','Name','Ticket'],axis=1,inplace=True )<categorify>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
label=LabelEncoder()
dataset['Sex']=label.fit_transform(dataset['Sex'])
 dataset['Age']=dataset['Age'].astype(int )<feature_engineering>
print("Starting Prediction by wikinews Embeddings for VAL")
 pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))
print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))
if score > best_score:
best_thresh = thresh
best_score = score
dataset['familyno']=dataset['SibSp']+dataset['Parch']+1<categorify>
print()
print("Best Tresh for wikinews is {0} at score {1}".format(best_thresh,best_score))
print()
pred_fasttext_val_y =(pred_fasttext_val_y>best_thresh )
dataset=pd.get_dummies(dataset,columns=['Pclass','Embarked'])
 dataset.drop(['SibSp','Parch'],axis=1,inplace=True)
 xtrain=dataset[:len(train)]
test=dataset[len(train):]
xtrain<predict_on_test>
print("Prediction for test based on wikinews Embeddings")
 pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_fasttext_test_y =(pred_fasttext_test_y>best_thresh)
 print("Wikinews Emebddings Prediction Complete" )
sky=GradientBoostingClassifier()
sky.fit(xtrain,y)
 c=sky.predict(test)
 submission['Survived']=c<save_to_csv>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
submission.to_csv('ver1.csv',index=False )<set_options>
print("Creating Model by paragram embeddings")
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
%%writefile main-RT.cpp
extern "C" {
};
using namespace std;
constexpr int n = 25;
constexpr int N = n * n;
using CW = vector<uint8_t>;
using TStamp = std::chrono::time_point<std::chrono::system_clock>;
struct task {
string id;
CW field;
int steps;
};
random_device rd;
mt19937 g(rd());
static atomic<bool> is_stopped;
static atomic<bool> is_timeouted;
double total_timeout = 8.5 * 60 * 60;
TStamp app_started = std::chrono::system_clock::now() ;
void terminate(int code){
is_stopped = true;
is_timeouted = true;
exit(code);
}
vector<string> split(std::string s, char delimiter){
s += delimiter;
vector<string> result;
size_t start = 0;
for(size_t i = start; i < s.size() ; ++i){
while(s[i] != delimiter){
++i;
}
result.push_back(s.substr(start, i - start)) ;
start = i + 1;
}
return result;
}
CW s2vec(const std::string& s){
CW v(N);
if(static_cast<int>(s.size())!= N + N - 1){
cerr << "wtf s2vec format len" << s.size() << endl;
terminate(1);
}
for(int i = 0; i < N; ++i){
v[i] = s[i + i] == '1';
}
return v;
}
string vec2s(const CW& v){
string res;
for(int i = 0; i < N; ++i){
if(v[i]){
res += "1,";
} else {
res += "0,";
}
}
res.resize(res.size() - 1);
return res;
}
int weight(const task& t){
int count = 0;
for(auto k : t.field){
if(k)++count;
}
return count;
}
vector<int> init_nums(int N){
vector<int> values;
for(int i = 0; i < N; ++i){
values.push_back(i);
}
return values;
}
const vector<int> n_nums = init_nums(N);
bool calc(const CW& field, int pos, int x, int y)noexcept {
static const int dx8[] = {-1, -1, -1, 0, 0, 1, 1, 1};
static const int dy8[] = {-1, 0, 1, -1, 1, -1, 0, 1};
int count = 0;
for(int i = 0; i < 8; ++i){
int a = x + dx8[i];
int b = y + dy8[i];
if(a < 0)a += n;
else if(a >= n)a -= n;
if(b < 0)b += n;
else if(b >= n)b -= n;
if(field[b * n + a]){
++count;
}
}
return(count | field[pos])== 3;
}
bool calc_left_line(const CW& field, size_t pos)noexcept {
static const size_t d_left8[] = { -1ULL, -25ULL, -24ULL, +24, +1, +25 + 24, +25, +26};
uint8_t count = 0;
for(int i = 0; i < 8; ++i){
count += field[pos + d_left8[i]];
}
return(count | field[pos])== 3;
}
bool calc_right_line(const CW& field, size_t pos)noexcept {
static const size_t d_right8[] = { -26ULL, -25ULL, - 49ULL, -1ULL, -24ULL, +24, +25, +1};
uint8_t count = 0;
for(int i = 0; i < 8; ++i){
count += field[pos + d_right8[i]];
}
return(count | field[pos])== 3;
}
bool calc_bottom_line(const CW& field, size_t pos)noexcept {
static const size_t d_bottom8[] = { -26ULL, -25ULL, -24ULL, -1ULL, 1, -599ULL, -600ULL, -601ULL};
uint8_t count = 0;
for(int i = 0; i < 8; ++i){
count += field[pos + d_bottom8[i]];
}
return(count | field[pos])== 3;
}
bool calc_top_line(const CW& field, size_t pos)noexcept {
static const size_t d_top8[] = { +599, +600, +601, -1ULL, 1, +24, +25, +26};
uint8_t count = 0;
for(int i = 0; i < 8; ++i){
count += field[pos + d_top8[i]];
}
return(count | field[pos])== 3;
}
bool calc_no_checks(const CW& field, size_t pos)noexcept {
static const size_t di8[] = {0, 1, 2, 25, 27, +50, +51, +52};
uint8_t count = 0;
for(int i = 0; i < 8; ++i){
count += field[pos + di8[i]];
}
return(count | field[pos + 26])== 3;
}
CW run_step(CW start)noexcept {
thread_local CW next(N);
for(int x = 1; x < n - 1; ++x){
next[x] = calc_top_line(start, x);
next[600 + x] = calc_bottom_line(start, 600 + x);
next[x * 25] = calc_left_line(start, x * 25);
next[x * 25 + 24] = calc_right_line(start, x * 25 + 24);
}
size_t pos = 0;
for(int y = 1; y < n - 1; ++y){
for(int x = 1; x < n - 1; ++x){
next[pos + 26] = calc_no_checks(start, pos);
++pos;
}
pos += 2;
}
for(auto i : {0, n-1, N - n, N - 1}){
next[i] = calc(start, i, i % n, i / n);
}
return next;
}
void run_step_partial(CW* steps, int x, int y, int step, vector<pair<int, int>>& changelog){
int range = step + 1;
for(int b = y - range; b <= y + range; ++b){
int q = b;
if(q < 0)q += n;
else if(q >= n)q -= n;
for(int a = x - range; a <= x + range; ++a){
int p = a;
if(p < 0)p += n;
else if(p >= n)p -= n;
size_t i = q * n + p;
bool not_vert_side = p > 0 && p + 1 < n;
bool not_hori_side = q > 0 && q + 1 < n;
bool value;
if(not_vert_side && not_hori_side){
value = calc_no_checks(steps[step], i - 26);
} else {
bool not_vert_side = p > 0 && p + 1 < n;
bool not_hori_side = q > 0 && q + 1 < n;
if(not_vert_side && q + 1 == n){
value = calc_bottom_line(steps[step], i);
} else if(not_vert_side && q == 0){
value = calc_top_line(steps[step], i);
} else if(not_hori_side && p == 0){
value = calc_left_line(steps[step], i);
} else if(not_hori_side && p + 1 == n){
value = calc_right_line(steps[step], i);
} else {
value = calc(steps[step], i, p, q);
}
}
bool previous = steps[step + 1][i];
if(value != previous){
changelog.push_back({step + 1, i});
steps[step + 1][i] = value;
}
}
}
}
int run_step_partial_last(CW* steps, int x, int y, int step, vector<pair<int, int>>& changelog, const CW& original){
int range = step + 1;
int diff = 0;
for(int b = y - range; b <= y + range; ++b){
int q = b;
if(q < 0)q += n;
else if(q >= n)q -= n;
for(int a = x - range; a <= x + range; ++a){
int p = a;
if(p < 0)p += n;
else if(p >= n)p -= n;
size_t i = q * n + p;
bool not_vert_side = p > 0 && p + 1 < n;
bool not_hori_side = q > 0 && q + 1 < n;
bool value;
if(not_vert_side && not_hori_side){
value = calc_no_checks(steps[step], i - 26);
} else {
if(not_vert_side && q + 1 == n){
value = calc_bottom_line(steps[step], i);
} else if(not_vert_side && q == 0){
value = calc_top_line(steps[step], i);
} else if(not_hori_side && p == 0){
value = calc_left_line(steps[step], i);
} else if(not_hori_side && p + 1 == n){
value = calc_right_line(steps[step], i);
} else {
value = calc(steps[step], i, p, q);
}
}
bool previous = steps[step + 1][i];
if(value != previous){
if(value == original[i]){
++diff;
} else {
--diff;
}
changelog.push_back({step + 1, i});
steps[step + 1][i] = value;
}
}
}
return diff;
}
void revert_changes(CW* steps, const vector<pair<int, int>>& changelog){
for(auto i : changelog){
steps[i.first][i.second] = !steps[i.first][i.second];
}
}
void print(const CW& field){
for(int pos = 0; pos < N; ++pos){
cout <<(field[pos] ? '*' : '.'); //static_cast<int>(field[pos]);
if(pos % n == n - 1){
cout << endl;
}
}
cout << endl;
}
int validate(CW start, const task& t){
for(int i = 0; i < t.steps; ++i){
start = run_step(start);
}
int count = 0;
for(int i = 0; i < N; ++i){
if(start[i] == t.field[i])
 ++count;
}
return count;
}
int validate_partial(const task& t, CW* steps, int x, int y, vector<pair<int, int>>& changelog){
changelog.clear() ;
for(int i = 0; i < t.steps - 1; ++i){
run_step_partial(steps, x, y, i, changelog);
}
int diff = run_step_partial_last(steps, x, y, t.steps - 1, changelog, t.field);
return diff;
}
static class Cache : private boost::noncopyable {
private:
map<string, pair<int, string>> cache;
mutex m;
public:
Cache() { }
auto get(const std::string& id){
lock_guard<mutex> g(m);
return cache[id];
}
bool set(const pair<int, CW>& value, const task& t){
const std::string& id = t.id;
{
lock_guard<mutex> g(m);
if(cache.find(id)!= cache.end() && cache[id].first >= value.first){
return false;
}
}
int score = validate(value.second, t);
if(score != value.first){
cerr << "wtf score for " << id << ": " << score << ' ' << value.first << endl;
for(auto i : value.second){
cerr <<(int)i << ' ';
}
cerr << endl;
throw std::runtime_error("wtf score");
}
string s = vec2s(value.second);
{
lock_guard<mutex> g(m);
cache[id] = {value.first, s};
ofstream cache_file("processed/cache-real-time", std::ofstream::app);
cache_file << id << ' ' << value.first << ' ' << s << endl;
return true;
}
}
} global_cache;
CW prepare_mask(CW solution, const task& t){
for(int i = 0; i < t.steps; ++i){
solution = run_step(solution);
}
vector<int> diff;
for(int i = 0; i < N; ++i){
if(t.field[i] != solution[i]){
diff.push_back(i);
}
}
for(int i = 0; i < t.steps; ++i){
vector<int> next_diff;
for(auto pos : diff){
next_diff.push_back(i);
int x = pos % n;
int y = pos / n;
static const int dx8[] = {-1, -1, -1, 0, 0, 1, 1, 1};
static const int dy8[] = {-1, 0, 1, -1, 1, -1, 0, 1};
for(int i = 0; i < 8; ++i){
int a = x + dx8[i];
int b = y + dy8[i];
if(a < 0)a += n;
else if(a >= n)a -= n;
if(b < 0)b += n;
else if(b >= n)b -= n;
next_diff.push_back(b * n + a);
}
}
diff = next_diff;
sort(diff.begin() , diff.end());
{
auto last = std::unique(diff.begin() , diff.end());
diff.erase(last, diff.end());
}
}
CW result(N);
for(auto i : diff){
result[i] = true;
}
// cerr << diff.size() << " for " << global_cache.get(t.id ).first << endl;
return result;
}
CW shake_single_if_equal(CW _field, const task& t, bool fast = false){
CW steps[t.steps + 1];
steps[0] = _field;
CW& field = steps[0];
vector<pair<int, int>> changelog;
int best_score = validate(field, t);
auto best_pos = field;
int score = best_score;
int current_score = best_score;
for(int i = 0; i < t.steps; ++i){
steps[i + 1] = run_step(steps[i]);
}
CW mask_to_shake = prepare_mask(best_pos, t);
int default_tries = fast ? 39 : 99;
int default_range = fast ? 2 : 3;
int tries = default_tries;
int range = default_range;
do {
if(score != best_score){
tries = default_tries;
score = best_score;
range = default_range;
}
if(tries % 50 == 0)range -= 1;
int shift = rand() % N;
bool dir = rand() % 2;
int i = shift;
int x = i % n;
int y = i / n;
for(int index = 0; index < N; ++index){
if(dir){
++i;
++x;
if(x == n){
x = 0;
++y;
if(y == n)y = 0;
}
if(i == N)i = 0;
} else {
--i;
--x;
if(x == -1){
x = n - 1;
--y;
if(y == -1)y = n - 1;
}
if(i == N)i = 0;
if(i < 0)i = N - 1;
}
if(!mask_to_shake[i] && tries % 5 != 0)continue; /// _---------- RT_ADJ
bool value = field[i];
field[i] = !field[i];
int new_score = current_score + validate_partial(t, steps, x, y, changelog);
if(new_score > best_score){
best_score = new_score;
best_pos = field;
global_cache.set({best_score, best_pos}, t);
} else if(new_score < best_score - range){
field[i] = !field[i];
} else {
if(rand() % 4 == 0){
field[i] = !field[i];
}
}
if(value == field[i]){
revert_changes(steps, changelog);
} else {
current_score = new_score;
}
}
} while(best_score != score || tries-- > 0);
return best_pos;
}
CW shake_k_and_single(const task &t, CW start, int tries, bool fast = false){
auto next_start = start;
vector<int> indexes = n_nums;
shuffle(indexes.begin() , indexes.end() , g);
for(int i = 0; i < tries; ++i){
next_start[indexes[i]] = !next_start[indexes[i]];
}
next_start = shake_single_if_equal(next_start, t, true);
return shake_single_if_equal(next_start, t, fast);
}
CW make_random() {
CW c(N);
auto indexes = n_nums;
shuffle(indexes.begin() , indexes.end() , g);
indexes.resize(N / 2);
for(auto i : indexes){
c[i] = 1;
}
return c;
}
CW make_checked() {
CW c(N);
for(size_t i = 0; i < N; i += 2){
c[i] = true;
}
return c;
}
namespace CNF_solver {
using CNF = std::vector<std::vector<int>>;
bool alive(const CW& field, int pos, int x, int y){
static const int dx8[] = {-1, -1, -1, 0, 0, 1, 1, 1};
static const int dy8[] = {-1, 0, 1, -1, 1, -1, 0, 1};
int count = 0;
for(int i = 0; i < 8; ++i){
int a = x + dx8[i];
int b = y + dy8[i];
if(a < 0)a += n;
else if(a >= n)a -= n;
if(b < 0)b += n;
else if(b >= n)b -= n;
if(field[b * n + a]){
++count;
}
}
return(count | field[pos])== 3;
}
vector<int> generate_positions(int x, int y){
static const int dx8[] = {-1, -1, -1, 0, 0, 1, 1, 1};
static const int dy8[] = {-1, 0, 1, -1, 1, -1, 0, 1};
vector<int> result;
result.push_back(-9999); // place holder
for(int i = 0; i < 8; ++i){
int a = x + dx8[i];
int b = y + dy8[i];
if(a < 0)a += n;
else if(a >= n)a -= n;
if(b < 0)b += n;
else if(b >= n)b -= n;
result.push_back(b * n + a);
}
return result;
}
// CNF: all values false: through negative
bool geti(uint32_t value, int index){
return(value &(1 << index)) ;
}
CNF universal_cnf() {
CNF result_cnf;
uint32_t mask = 0;
while(!geti(mask, 10)) {
int count = 0;
for(int i = 0; i < 8; ++i){
if(geti(mask, i)) {
++count;
}
}
bool alive =(count | geti(mask, 8)) == 3;
bool is_true =(alive == geti(mask, 9)) ;
if(!is_true){
vector<int> next;
for(int i = 0; i <= 9; ++i){
if(geti(mask, i)) {
next.push_back(-i - 1);
} else {
next.push_back(i + 1);
}
}
result_cnf.push_back(next);
}
sort(result_cnf.begin() , result_cnf.end());
++mask;
}
// 325065 CNF -> 160k CNF
// speed increased 1.6 ; 30 ->
// optimize_cnf(result_cnf);
// Optimized much better externally by python tool sympy:
// $ python3
// >>>
// >>> x, y, z = symbols('x y z')
 // >>> simplify_logic(( x & y & ~z)|(x & ~y & z), form="cnf")
 // x &(y | z)&(~y | ~z)
 result_cnf = {{1, 2, 3, 4, 5, 6, 7, -10}, {1, 2, 3, 4, 5, 6, 8, -10}, {1, 2, 3, 4, 5, 6, 9, -10}, {1, 2, 3, 4, 5, 7, 8, -10}, {1, 2, 3, 4, 5, 7, 9, -10}, {1, 2, 3, 4, 5, 8, 9, -10}, {1, 2, 3, 4, 6, 7, 8, -10}, {1, 2, 3, 4, 6, 7, 9, -10}, {1, 2, 3, 4, 6, 8, 9, -10}, {1, 2, 3, 4, 7, 8, 9, -10}, {1, 2, 3, 5, 6, 7, 8, -10}, {1, 2, 3, 5, 6, 7, 9, -10}, {1, 2, 3, 5, 6, 8, 9, -10}, {1, 2, 3, 5, 7, 8, 9, -10}, {1, 2, 3, 6, 7, 8, 9, -10}, {1, 2, 4, 5, 6, 7, 8, -10}, {1, 2, 4, 5, 6, 7, 9, -10}, {1, 2, 4, 5, 6, 8, 9, -10}, {1, 2, 4, 5, 7, 8, 9, -10}, {1, 2, 4, 6, 7, 8, 9, -10}, {1, 2, 5, 6, 7, 8, 9, -10}, {1, 3, 4, 5, 6, 7, 8, -10}, {1, 3, 4, 5, 6, 7, 9, -10}, {1, 3, 4, 5, 6, 8, 9, -10}, {1, 3, 4, 5, 7, 8, 9, -10}, {1, 3, 4, 6, 7, 8, 9, -10}, {1, 3, 5, 6, 7, 8, 9, -10}, {1, 4, 5, 6, 7, 8, 9, -10}, {2, 3, 4, 5, 6, 7, 8, -10}, {2, 3, 4, 5, 6, 7, 9, -10}, {2, 3, 4, 5, 6, 8, 9, -10}, {2, 3, 4, 5, 7, 8, 9, -10}, {2, 3, 4, 6, 7, 8, 9, -10}, {2, 3, 5, 6, 7, 8, 9, -10}, {2, 4, 5, 6, 7, 8, 9, -10}, {3, 4, 5, 6, 7, 8, 9, -10}, {-1, -10, -2, -3, -4}, {-1, -10, -2, -3, -5}, {-1, -10, -2, -3, -6}, {-1, -10, -2, -3, -7}, {-1, -10, -2, -3, -8}, {-1, -10, -2, -4, -5}, {-1, -10, -2, -4, -6}, {-1, -10, -2, -4, -7}, {-1, -10, -2, -4, -8}, {-1, -10, -2, -5, -6}, {-1, -10, -2, -5, -7}, {-1, -10, -2, -5, -8}, {-1, -10, -2, -6, -7}, {-1, -10, -2, -6, -8}, {-1, -10, -2, -7, -8}, {-1, -10, -3, -4, -5}, {-1, -10, -3, -4, -6}, {-1, -10, -3, -4, -7}, {-1, -10, -3, -4, -8}, {-1, -10, -3, -5, -6}, {-1, -10, -3, -5, -7}, {-1, -10, -3, -5, -8}, {-1, -10, -3, -6, -7}, {-1, -10, -3, -6, -8}, {-1, -10, -3, -7, -8}, {-1, -10, -4, -5, -6}, {-1, -10, -4, -5, -7}, {-1, -10, -4, -5, -8}, {-1, -10, -4, -6, -7}, {-1, -10, -4, -6, -8}, {-1, -10, -4, -7, -8}, {-1, -10, -5, -6, -7}, {-1, -10, -5, -6, -8}, {-1, -10, -5, -7, -8}, {-1, -10, -6, -7, -8}, {-10, -2, -3, -4, -5}, {-10, -2, -3, -4, -6}, {-10, -2, -3, -4, -7}, {-10, -2, -3, -4, -8}, {-10, -2, -3, -5, -6}, {-10, -2, -3, -5, -7}, {-10, -2, -3, -5, -8}, {-10, -2, -3, -6, -7}, {-10, -2, -3, -6, -8}, {-10, -2, -3, -7, -8}, {-10, -2, -4, -5, -6}, {-10, -2, -4, -5, -7}, {-10, -2, -4, -5, -8}, {-10, -2, -4, -6, -7}, {-10, -2, -4, -6, -8}, {-10, -2, -4, -7, -8}, {-10, -2, -5, -6, -7}, {-10, -2, -5, -6, -8}, {-10, -2, -5, -7, -8}, {-10, -2, -6, -7, -8}, {-10, -3, -4, -5, -6}, {-10, -3, -4, -5, -7}, {-10, -3, -4, -5, -8}, {-10, -3, -4, -6, -7}, {-10, -3, -4, -6, -8}, {-10, -3, -4, -7, -8}, {-10, -3, -5, -6, -7}, {-10, -3, -5, -6, -8}, {-10, -3, -5, -7, -8}, {-10, -3, -6, -7, -8}, {-10, -4, -5, -6, -7}, {-10, -4, -5, -6, -8}, {-10, -4, -5, -7, -8}, {-10, -4, -6, -7, -8}, {-10, -5, -6, -7, -8}, {1, 10, 2, 3, 4, 5, -6, -7, -8}, {1, 10, 2, 3, 4, 6, -5, -7, -8}, {1, 10, 2, 3, 4, 7, -5, -6, -8}, {1, 10, 2, 3, 4, 8, -5, -6, -7}, {1, 10, 2, 3, 5, 6, -4, -7, -8}, {1, 10, 2, 3, 5, 6, -4, -7, -9}, {1, 10, 2, 3, 5, 6, -7, -8, -9}, {1, 10, 2, 3, 5, 7, -4, -6, -8}, {1, 10, 2, 3, 5, 7, -4, -6, -9}, {1, 10, 2, 3, 5, 7, -4, -8, -9}, {1, 10, 2, 3, 5, 7, -6, -8, -9}, {1, 10, 2, 3, 5, 8, -4, -6, -7}, {1, 10, 2, 3, 5, 8, -6, -7, -9}, {1, 10, 2, 3, 6, 7, -4, -5, -8}, {1, 10, 2, 3, 6, 7, -4, -5, -9}, {1, 10, 2, 3, 6, 7, -5, -8, -9}, {1, 10, 2, 3, 6, 8, -4, -5, -7}, {1, 10, 2, 3, 6, 8, -5, -7, -9}, {1, 10, 2, 3, 7, 8, -4, -5, -6}, {1, 10, 2, 3, 7, 8, -5, -6, -9}, {1, 10, 2, 4, 5, 6, -3, -7, -8}, {1, 10, 2, 4, 5, 7, -3, -6, -8}, {1, 10, 2, 4, 5, 8, -3, -6, -7}, {1, 10, 2, 4, 6, 7, -3, -5, -8}, {1, 10, 2, 4, 6, 8, -3, -5, -7}, {1, 10, 2, 4, 7, 8, -3, -5, -6}, {1, 10, 2, 5, 6, 7, -3, -4, -8}, {1, 10, 2, 5, 6, 7, -3, -4, -9}, {1, 10, 2, 5, 6, 7, -3, -8, -9}, {1, 10, 2, 5, 6, 8, -3, -4, -7}, {1, 10, 2, 5, 6, 8, -3, -7, -9}, {1, 10, 2, 5, 7, 8, -3, -4, -6}, {1, 10, 2, 5, 7, 8, -3, -6, -9}, {1, 10, 2, 6, 7, 8, -3, -4, -5}, {1, 10, 2, 6, 7, 8, -3, -5, -9}, {1, 10, 3, 4, 5, 6, -2, -7, -8}, {1, 10, 3, 4, 5, 7, -2, -6, -8}, {1, 10, 3, 4, 5, 8, -2, -6, -7}, {1, 10, 3, 4, 6, 7, -2, -5, -8}, {1, 10, 3, 4, 6, 8, -2, -5, -7}, {1, 10, 3, 4, 7, 8, -2, -5, -6}, {1, 10, 3, 5, 6, 7, -2, -4, -8}, {1, 10, 3, 5, 6, 7, -2, -4, -9}, {1, 10, 3, 5, 6, 7, -2, -8, -9}, {1, 10, 3, 5, 6, 8, -2, -4, -7}, {1, 10, 3, 5, 6, 8, -2, -7, -9}, {1, 10, 3, 5, 7, 8, -2, -4, -6}, {1, 10, 3, 5, 7, 8, -2, -6, -9}, {1, 10, 3, 6, 7, 8, -2, -4, -5}, {1, 10, 3, 6, 7, 8, -2, -5, -9}, {1, 10, 4, 5, 6, 7, -2, -3, -8}, {1, 10, 4, 5, 6, 8, -2, -3, -7}, {1, 10, 4, 5, 7, 8, -2, -3, -6}, {1, 10, 4, 6, 7, 8, -2, -3, -5}, {1, 10, 5, 6, 7, 8, -2, -3, -4}, {1, 10, 5, 6, 7, 8, -2, -3, -9}, {10, 2, 3, 4, 5, 6, -1, -7, -8}, {10, 2, 3, 4, 5, 7, -1, -6, -8}, {10, 2, 3, 4, 5, 8, -1, -6, -7}, {10, 2, 3, 4, 6, 7, -1, -5, -8}, {10, 2, 3, 4, 6, 8, -1, -5, -7}, {10, 2, 3, 4, 7, 8, -1, -5, -6}, {10, 2, 3, 5, 6, 7, -1, -4, -8}, {10, 2, 3, 5, 6, 7, -1, -4, -9}, {10, 2, 3, 5, 6, 7, -1, -8, -9}, {10, 2, 3, 5, 6, 8, -1, -4, -7}, {10, 2, 3, 5, 6, 8, -1, -7, -9}, {10, 2, 3, 5, 7, 8, -1, -4, -6}, {10, 2, 3, 5, 7, 8, -1, -6, -9}, {10, 2, 3, 6, 7, 8, -1, -4, -5}, {10, 2, 3, 6, 7, 8, -1, -5, -9}, {10, 2, 4, 5, 6, 7, -1, -3, -8}, {10, 2, 4, 5, 6, 8, -1, -3, -7}, {10, 2, 4, 5, 7, 8, -1, -3, -6}, {10, 2, 4, 6, 7, 8, -1, -3, -5}, {10, 2, 5, 6, 7, 8, -1, -3, -4}, {10, 2, 5, 6, 7, 8, -1, -3, -9}, {10, 3, 4, 5, 6, 7, -1, -2, -8}, {10, 3, 4, 5, 6, 8, -1, -2, -7}, {10, 3, 4, 5, 7, 8, -1, -2, -6}, {10, 3, 4, 6, 7, 8, -1, -2, -5}, {10, 3, 5, 6, 7, 8, -1, -2, -4}, {10, 3, 5, 6, 7, 8, -1, -2, -9}, {10, 4, 5, 6, 7, 8, -1, -2, -3}};
return result_cnf;
}
// 0...N-1: previous position
// N...2N - 1: current position
vector<std::vector<int>> cnf_universal() {
// universal "conways rules" CNF
CNF cnf;
CNF cnf_conway_cell = universal_cnf() ;
for(int y = 0; y < n; ++y){
for(int x = 0; x < n; ++x){
int pos_previous = y * n + x;
int pos_current = y * n + x + N;
vector<int> positions = generate_positions(x, y);
positions.push_back(pos_previous);
positions.push_back(pos_current);
for(auto& c : cnf_conway_cell){
vector<int> next;
for(auto& p : c){
if(p < 0){
next.push_back(-positions[-p] - 1);
} else {
next.push_back(positions[p] + 1);
}
}
cnf.push_back(next);
}
}
}
cout << "CNF size: " << cnf.size() << endl;
return cnf;
}
const static CNF cnf_conway_rules = cnf_universal() ;
CNF get_cnf(const task& t, const std::pair<int, int>& ignore_range = {0, 0}){
int k = t.steps;
if(k == 1){
int k = t.steps;
CNF cnf = cnf_conway_rules;
for(int iter = 1; iter < k; ++iter){
CNF cnf_i = cnf_conway_rules;
for(auto& i : cnf_i){
for(auto& j : i){
if(j > 0)j += N * iter;
else j -= N * iter;
}
cnf.push_back(i);
}
}
// fulfill CNF for current field
for(int i = 0; i < N; ++i){
int var = i + k * N + 1;
if(t.field[i]){
cnf.push_back({var});
} else {
cnf.push_back({-var});
}
}
return cnf;
}
// First level
CNF cnf = cnf_conway_rules;
// Middle levels: not first, not last
for(int iter = 1; iter + 1 < k; ++iter){
CNF cnf_i = cnf_conway_rules;
for(auto& i : cnf_i){
for(auto& j : i){
if(j > 0)j += N * iter;
else j -= N * iter;
}
cnf.push_back(i);
}
}
// fulfill CNF for current field
CNF cnf_last = cnf_conway_rules;
for(auto& i : cnf_last){
int to_ignore = 0;
for(auto& j : i){
for(int l = N + 1; l <= N + N; ++l){
if(j == l || j == -l){
to_ignore = j;
}
}
}
int pos = -1;
if(to_ignore > 0){
pos = to_ignore - N - 1;
if(t.field[pos])continue;
}
if(to_ignore < 0){
pos = -to_ignore - N - 1;
if(!t.field[-to_ignore - N - 1])continue;
}
if(pos >= ignore_range.first && pos < ignore_range.second){
continue;
}
std::vector<int> c;
for(auto& j : i){
if(j == to_ignore)continue;
if(j > 0)
 c.push_back(j + N *(k - 1)) ;
else
c.push_back(j - N *(k - 1)) ;
}
cnf.push_back(c);
}
return cnf;
}
} // namespace CNF_solver
struct KissatSolver {
KissatSolver(double timeout = 2.1)
 : timeout(timeout)
 , cleaner([this]() -> void {
cleanup() ;
})
 {
}
~KissatSolver() {
cleaner.join() ;
}
kissat* add(string id){
kissat* solver = kissat_init() ;
{
lock_guard<mutex> g(m);
if(all.find(id)!= all.end())throw std::runtime_error("Kissat solver already contains" + id);
all[id] = {solver, std::chrono::system_clock::now() };
}
return solver;
}
void notify_done(string id){
lock_guard<mutex> g(m);
to_free.insert(id);
}
void cleanup() {
while(!is_stopped){
this_thread::sleep_for(300ms);
lock_guard<mutex> g(m);
int to_clean = 0;
for(const auto& id : to_free){
auto& item = all[id];
kissat* solver = item.first;
kissat_release(solver);
all.erase(id);
++to_clean;
}
to_free.clear() ;
int to_terminate = 0;
TStamp now = std::chrono::system_clock::now() ;
for(const auto& i : all){
auto item = i.second;
double running = std::chrono::duration_cast<std::chrono::seconds>(now - item.second ).count() ;
if(running > timeout){
kissat_terminate(item.first);
++to_terminate;
}
}
if(to_terminate || to_clean){
cout << "terminated " << to_terminate << " cleaned " << to_clean << endl;
}
}
}
private:
double timeout;
map<string, std::pair<kissat*, TStamp>> all;
thread cleaner;
set<string> to_free;
mutex m;
} kissat_solver;
void sat_solver_k(const task& t){
int k = t.steps;
CNF_solver::CNF cnf = CNF_solver::get_cnf(t);
kissat* solver = kissat_solver.add(t.id);
kissat_set_option(solver, "quiet", 1);
kissat_set_configuration(solver, "sat");
kissat_reserve(solver, N * k);
for(auto c : cnf){
for(auto i : c){
kissat_add(solver, i);
}
kissat_add(solver, 0);
}
int result = kissat_solve(solver);
switch(result){
case 20:
cerr << "wtf solver failed for " << t.id << endl;
break;
}
CW solution;
bool at_least_one_true = 0;
for(int i = 1; i <= N; ++i){
int next = kissat_value(solver, i);
if(next > 0){
solution.push_back(1);
at_least_one_true = true;
} else {
solution.push_back(0);
}
}
kissat_solver.notify_done(t.id);
int score = validate(solution, t);
global_cache.set({score, solution}, t);
if(score != N && at_least_one_true){
shake_single_if_equal(solution, t);
}
}
bool solve_sat(const task& t){
auto it = global_cache.get(t.id);
if(is_timeouted)return false;
if(it.first == N){
return true;
}
if(t.steps == 1){
sat_solver_k(t);
} else if(t.steps == 2 && weight(t)< 100){ /// ------------ RT_ADJ
sat_solver_k(t);
}
return global_cache.get(t.id ).first == N;
}
bool solve_fast(const task& t){
if(is_timeouted)return false;
auto it = global_cache.get(t.id);
if(it.first == N){
return true;
}
{
if(weight(t)< 200){
shake_single_if_equal(CW(N), t, true);
} else {
// shake_single_if_equal(make_random() , t, true);
}
shake_single_if_equal(make_checked() , t, true);
if(t.steps % 2 == 0)
 shake_single_if_equal(t.field, t, true);
else
shake_single_if_equal(run_step(t.field), t, true);
}
// enabled this only as efficient
int score = it.first;
size_t iters = 5;
CW next = s2vec(it.second);
while(iters--){
CW next2 = shake_single_if_equal(next, t, true);
if(validate(next2, t)>= validate(next, t)) {
next = next2;
}
it = global_cache.get(t.id);
int new_score = it.first;
if(new_score == N)break;
if(new_score != score){
score = new_score;
next = s2vec(it.second);
iters = 11;
}
}
return global_cache.get(t.id ).first == N;
}
bool solve_slower(const task& t){
if(is_timeouted)return false;
auto it = global_cache.get(t.id);
if(it.first == N){
return true;
}
// enabled this only as efficient
int score = it.first;
size_t iters = 4;
CW next = s2vec(it.second);
while(iters--){
CW next2 = shake_single_if_equal(next, t);
CW next3 = shake_k_and_single(t, next2, 8);
if(validate(next3, t)>= validate(next, t)) {
next = next3;
}
if(validate(next2, t)>= validate(next, t)) {
next = next2;
}
it = global_cache.get(t.id);
int new_score = it.first;
if(new_score == N)break;
if(new_score != score){
score = new_score;
next = s2vec(it.second);
iters = 6;
}
}
/*
for(int i = 0; i < 4; ++i){
CW next = make_random() ;
for(int j = 0; j < 5; ++j){
next = shake_k_and_single(t, next, 30);
}
}
*/
// trying to run upfront far
// run_upfront(id, end, steps);
return global_cache.get(t.id ).first == N;
}
void validate_training(const std::string& file){
ifstream input(file);
string line;
input >> line;
while(input >> line){
vector<string> v = split(line, ',');
if(v.size() != static_cast<size_t>(N + N + 2)) {
cerr << "incorrect input within line within validate: " << line << endl;
terminate(1);
}
string id = v[0];
int steps = stoi(v[1]);
CW start(N);
CW end(N);
for(int i = 0; i < N; ++i){
start[i] =(v[i + 2] == "1");
end[i] =(v[N + i + 2] == "1");
}
if(validate(start, {id, end, steps})!= N){
cerr << "validate failed at: " << id << endl;
terminate(1);
}
}
}
pair<int, int> calc_stats(const std::vector<task>& tasks){
int count = 0;
int count_solved = 0;
int total_score = 0;
for(const task & t : tasks){
++count;
int score = global_cache.get(t.id ).first;
total_score += score;
if(score == N)++count_solved;
}
int max_score = count * N;
cout << "Solved: " << count_solved << "/" << count ;
cout << " :<>: " << total_score << "/" << max_score << " = " << static_cast<double>(total_score)/ max_score << endl;;
return {count_solved, total_score};
}
void print_stats(const vector<task>& tasks){
auto started = std::chrono::system_clock::now() ;
auto first_stats = calc_stats(tasks);
while(!is_stopped){
sleep(9);
{ // time-out for application ?
TStamp now = std::chrono::system_clock::now() ;
double total_running = std::chrono::duration_cast<std::chrono::seconds>(now - app_started ).count() ;
if(total_running > total_timeout){
is_timeouted = true;
}
}
double running =(std::chrono::system_clock::now() - started ).count() ;
auto current = calc_stats(tasks);
double speed1 =(current.first - first_stats.first)/ running * 1e9;
double speed2 =(current.second - first_stats.second)/ running * 1e9;
cout << "speed " << std::fixed << std::setprecision(6)<< speed1 << ' ' << speed2 << ' ';
}
calc_stats(tasks);
}
int main(int argc, char *argv[]){
validate_training("train.csv");
if(argc > 1 && strcmp(argv[1], "--only-validate")== 0){
cerr << "Validated train!" << endl;
terminate(0);
}
if(argc > 1 && strcmp(argv[1], "--timeout")== 0){
total_timeout = stoi(argv[2]);
}
ifstream input("test.csv");
string line;
// header
input >> line;
std::vector<task> all_tasks;
while(input >> line){
vector<string> v = split(line, ',');
if(v.size() != static_cast<size_t>(N + 2)) {
cerr << "incorrect input within line: " << line << endl;
terminate(1);
}
string id = v[0];
int steps = stoi(v[1]);
// if(id >= "51000")continue;
// if(steps != 3)continue;
CW field(N);
for(int i = 0; i < N; ++i){
field[i] = v[i + 2] == "1";
}
task t{id, field, steps};
CW empty(N);
global_cache.set({validate(empty, t), empty}, t);
all_tasks.push_back(t);
}
if(argc > 1 && strcmp(argv[1], "--shuffle")== 0){
cout << "shuffle tasks randomly" << endl;
shuffle(all_tasks.begin() , all_tasks.end() , g);
}
if(argc > 1 && strcmp(argv[1], "--sort-score")== 0){
cout << "sort by score" << endl;
sort(all_tasks.begin() , all_tasks.end() , [](const auto& i, const auto& j){
int score_a = global_cache.get(i.id ).first;
int score_b = global_cache.get(j.id ).first;
return score_a < score_b;
});
}
if(argc > 1 && strcmp(argv[1], "--sort-score-reverse")== 0){
cout << "sort by score(reverse" << endl;
sort(all_tasks.begin() , all_tasks.end() , [](const auto& i, const auto& j){
int score_a = global_cache.get(i.id ).first;
int score_b = global_cache.get(j.id ).first;
return score_a > score_b;
});
}
if(argc > 1 && strcmp(argv[1], "--sort-weight")== 0){
cout << "sort by weight" << endl;
sort(all_tasks.begin() , all_tasks.end() , [](const auto& i, const auto& j){
return(weight(i)< weight(j)) ;
});
}
cout << "solve SAT" << endl;
{
boost::asio::thread_pool pool(std::thread::hardware_concurrency());
for(const auto& t : all_tasks){
if(global_cache.get(t.id ).first == N)continue;
boost::asio::post(pool, [t]() {
solve_sat(t);
} 
);
}
thread stats([all_tasks]() {print_stats(all_tasks); });
pool.join() ;
is_stopped = true;
stats.join() ;
}
cout << "solve fast" << endl;
{
is_stopped = false;
boost::asio::thread_pool pool(std::thread::hardware_concurrency());
for(const auto& t : all_tasks){
if(global_cache.get(t.id ).first == N)continue;
boost::asio::post(pool, [t]() {
solve_fast(t);
} 
);
}
thread stats([all_tasks]() {print_stats(all_tasks); });
pool.join() ;
is_stopped = true;
stats.join() ;
}
while(!is_timeouted){
cout << "solve slow" << endl;
sort(all_tasks.begin() , all_tasks.end() , [](const auto& i, const auto& j){
return(weight(i)> weight(j)) ;
});
is_stopped = false;
boost::asio::thread_pool pool(std::thread::hardware_concurrency());
for(const auto& t : all_tasks){
if(global_cache.get(t.id ).first == N)continue;
boost::asio::post(pool, [t]() {
solve_slower(t);
} 
);
}
thread stats([all_tasks]() {print_stats(all_tasks); });
pool.join() ;
is_stopped = true;
stats.join() ;
}
}<install_modules>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
!apt-get install libboost-all-dev -y 
<load_pretrained>
print("Starting Prediction by paragram Embeddings")
 pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))
print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))
if score > best_score:
best_thresh = thresh
best_score = score
if KAGGLE:
pass
else:
!g++ -o main-RT main-RT.cpp -std=c++17 -g -O3 -lpthread -Wall -Wextra -march=native -funroll-loops -fno-omit-frame-pointer kissat/build/libkissat.a kissat/build/handle.o<feature_engineering>
print()
print("Best Tresh for paragram is {0} at score {1}".format(best_thresh,best_score))
print()
pred_paragram_val_y =(pred_paragram_val_y>best_thresh )
%%writefile dump.cpp
using namespace std;
int N = 625;
int main() {
map<string, pair<int, string>> cache;
{
ifstream cache_file("processed/cache");
string id;
int score;
string field;
while(cache_file >> id >> score >> field){
auto it = cache[id];
if(it.first < score){
cache[id] = {score, field};
}
}
}
string time_suffix = to_string(time(nullptr)) ;
ofstream dump_file("processed/dump-" + time_suffix + ".csv");
ofstream shrink_cache("processed/shrink-cache-" + time_suffix);
dump_file << "id";
for(int i = 0; i < N; ++i){
dump_file << ",start_" << i;
}
dump_file << '
';
for(auto i : cache){
dump_file << i.first << "," << i.second.second << '
';
shrink_cache << i.first << ' ' << i.second.first << ' ' << i.second.second << '
';
}
}<import_modules>
print("Prediction for test based on paragram Embeddings")
 pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_paragram_test_y =(pred_paragram_test_y>best_thresh)
 print("Paragram Emebddings Prediction Complete" )
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import clear_output
import pandas as pd
from PIL import Image
import time
import math
from tqdm.auto import tqdm<import_modules>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
print(sys.version )<import_modules>
pred_val_y = 0.34*pred_glove_val_y + 0.33*pred_fasttext_val_y + 0.33*pred_paragram_val_y
best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))
print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))
if score > best_score:
best_thresh = thresh
best_score = score
import tensorflow as tf<import_modules>
print()
print("Best Tresh for overall is {0} at score {1}".format(best_thresh,best_score))
print()
pred_val_y =(pred_val_y>best_thresh )
tf.__version__<define_variables>
pred_test_y = 0.34*pred_glove_test_y + 0.33*pred_fasttext_test_y + 0.33*pred_paragram_test_y
pred_test_y =(pred_test_y>best_thresh ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )
SIZE = 25<normalization>
print("Submission Created" )