Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<train_model>
import re
import time
import gc
import random
import os
import numpy as np
import pandas as pd
from tqdm import tqdm
from tabulate import tabulate
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, roc_auc_score
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import torch
import torch.nn as nn
import torch.utils.data
history = model.fit(
 get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback],
validation_data=None if SKIP_VALIDATION else get_validation_dataset()
)<choose_model_class>
if not os.path.exists('./embeddings'):
os.mkdir('./embeddings')
 os.mkdir('./embeddings/glove.840B.300d/')
 with zipfile.ZipFile('.. /input/quora-insincere-questions-classification/embeddings.zip', 'r')as z:
with z.open('glove.840B.300d/glove.840B.300d.txt')as zf, open('./embeddings/glove.840B.300d/glove.840B.300d.txt', 'wb')as f:
shutil.copyfileobj(zf, f )
with strategy.scope() :
rnet = DenseNet201(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
)
 model2 = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model2.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model2.summary()<train_model>
embed_size = 300
max_features = 30000
maxlen = 72
batch_size = 512
train_epochs = 5
SEED = 1029
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
history2 = model2.fit(
 get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback],
validation_data=None if SKIP_VALIDATION else get_validation_dataset()
)<predict_on_test>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
x = re.sub('[0-9]{1}', '
return x
mispell_dict = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will",
"how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am",
"I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am",
"i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have",
"it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not",
"mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not",
"needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will",
"she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have",
"so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is",
"there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would",
"they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have",
"to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have",
"we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are",
"what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is",
"why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not",
"wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are",
"y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are",
"you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling',
'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize',
'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are',
'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does',
'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist',
'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what',
'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 spaces_re = re.compile('\s')
 number_re, numbers_re = re.compile('(?<![0-9])[0-9]{1}(?![0-9])'), re.compile('[0-9]{2,}')
 CAPS_re = re.compile('[A-Z]{2,}')
 punct_re = re.compile('(\?|!){2,}')
 def prop_pattern(text, pattern_re):
return len(pattern_re.findall(text)) /(len(spaces_re.findall(text)) +1)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )
if not SKIP_VALIDATION:
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
m = model.predict(images_ds)
 m2 = model2.predict(images_ds)
 scores = []
for alpha in np.linspace(0,1,100):
cm_probabilities = alpha*m+(1-alpha)*m2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 scores.append(f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro'))
print("Correct labels: ", cm_correct_labels.shape, cm_correct_labels)
 print("Predicted labels: ", cm_predictions.shape, cm_predictions)
 plt.plot(scores)
 best_alpha = np.argmax(scores)/100
cm_probabilities = best_alpha*m+(1-best_alpha)*m2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 else:
best_alpha = 0.44<define_variables>
train_df = pd.read_csv(".. /input/quora-insincere-questions-classification/train.csv")
 test_df = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv" )
if not SKIP_VALIDATION:
val_batch = iter(cmdataset.unbatch().batch(1))
noip = sum(cm_predictions!=cm_correct_labels)
 print('Number of incorrect predictions: ' + str(noip)+ '('+str(round(noip/NUM_VALIDATION_IMAGES*100,1)) +'%)')
 for fi in range(NUM_VALIDATION_IMAGES):
x = next(val_batch)
 if cm_predictions[fi] != cm_correct_labels[fi]:
display_batch_of_images(x,np.array([cm_predictions[fi]]),figsize = 2 )<compute_test_metric>
description = train_df['target'].value_counts(sort=False)
 print(tabulate([['Count',description[0],description[1]],
['Proportion',"{0:.0%}".format(description[0]/sum(description)) ,"{0:.0%}".format(description[1]/sum(description)) ]],
headers=['',description.index[0], description.index[1]]))
del description
if not SKIP_VALIDATION:
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))<save_to_csv>
words_nb_df = train_df.copy()
words_nb_df['word_nb'] = words_nb_df["question_text"].apply(lambda x : len(spaces_re.findall(x)))
 print(tabulate([['Mean',words_nb_df[words_nb_df['target']==0]['word_nb'].mean() ,words_nb_df[words_nb_df['target']==1]['word_nb'].mean() ],
['25%',words_nb_df[words_nb_df['target']==0]['word_nb'].quantile(0.25),words_nb_df[words_nb_df['target']==1]['word_nb'].quantile(0.25)],
['Med',words_nb_df[words_nb_df['target']==0]['word_nb'].median() ,words_nb_df[words_nb_df['target']==1]['word_nb'].median() ],
['75%',words_nb_df[words_nb_df['target']==0]['word_nb'].quantile(0.75),words_nb_df[words_nb_df['target']==1]['word_nb'].quantile(0.75)]],
headers=['','Sincere', 'Insincere']))
del words_nb_df
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = best_alpha*model.predict(test_images_ds)+(1-best_alpha)*model2.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<define_search_space>
pattern_list = [number_re, numbers_re, CAPS_re, punct_re, mispellings_re]
legends = ["Number", "Numbers", "CAPS", "Punctuation", "Mispellings"]
display = []
sincere_df = train_df[train_df['target']==0]["question_text"].copy()
insincere_df = train_df[train_df['target']==1]["question_text"].copy()
for i in range(len(legends)) :
if legends[i] == "Mispellings" :
sincere_df = sincere_df.apply(lambda x: x.lower())
 insincere_df = insincere_df.apply(lambda x: x.lower())
 metrics_sincere = sincere_df.apply(lambda x: prop_pattern(x, pattern_list[i]))
metrics_insincere = insincere_df.apply(lambda x: prop_pattern(x, pattern_list[i]))
display.append([legends[i] + " - Prop(Words)", "{0:.1%}".format(metrics_sincere.mean()),"{0:.1%}".format(metrics_insincere.mean())])
 display.append([legends[i] + " - Prop(Texts)", "{0:.1%}".format(sum(metrics_sincere>0)/len(metrics_sincere)) ,"{0:.1%}".format(sum(metrics_insincere>0)/len(metrics_insincere)) ])
 print(tabulate(display, headers=['','Sincere', 'Insincere']))
del sincere_df, insincere_df, metrics_sincere, metrics_insincere, train_df, test_df
%%writefile main.cpp
using namespace std;
using namespace std::chrono;
int N_JOBS = 4;
int END_TIME = 10;//in minutes
auto START_TIME = high_resolution_clock::now() ;
constexpr array<uint8_t, 15> DISTRIBUTION{2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 5}; // You can setup how many families you need for swaps and what best choice use for each family
// {2, 5} it's mean the first random family will brute force for choices 1-2 and the second random family will brute force for choices 1-5
constexpr int MAX_OCCUPANCY = 300;
constexpr int MIN_OCCUPANCY = 125;
constexpr int BEST_N = 10;
array<uint8_t, 5000> n_people;
array<array<uint8_t, 10>, 5000> choices;
array<array<uint16_t, 10>, 5000> PCOSTM;
array<array<double, 176>, 176> ACOSTM;
struct Index {
Index(array<uint8_t, 5000> assigned_days_): assigned_days(assigned_days_){
setup() ;
}
array<uint8_t, 5000> assigned_days;
array<uint16_t, 100> daily_occupancy_{};
int preference_cost_ = 0;
void setup() {
preference_cost_ = 0;
daily_occupancy_.fill(0);
for(int j = 0; j < assigned_days.size() ; ++j){
daily_occupancy_[choices[j][assigned_days[j]]] += n_people[j];
preference_cost_ += PCOSTM[j][assigned_days[j]];
}
}
double calc(const array<uint16_t, 5000>& indices, const array<uint8_t, DISTRIBUTION.size() >& change){
double accounting_penalty = 0.0;
auto daily_occupancy = daily_occupancy_;
int preference_cost = preference_cost_;
for(int i = 0; i < DISTRIBUTION.size() ; ++i){
int j = indices[i];
daily_occupancy[choices[j][assigned_days[j]]] -= n_people[j];
daily_occupancy[choices[j][ change[i]]] += n_people[j];
preference_cost += PCOSTM[j][change[i]] - PCOSTM[j][assigned_days[j]];
}
for(auto occupancy : daily_occupancy)
 if(occupancy < MIN_OCCUPANCY)
 return 1e12*(MIN_OCCUPANCY-occupancy);
else if(occupancy > MAX_OCCUPANCY)
 return 1e12*(occupancy - MAX_OCCUPANCY);
for(int day = 0; day < 99; ++day)
 accounting_penalty += ACOSTM[daily_occupancy[day]-125][daily_occupancy[day+1]-125];
accounting_penalty += ACOSTM[daily_occupancy[99]-125][daily_occupancy[99]-125];
return preference_cost + accounting_penalty;
}
void reindex(const array<uint16_t, DISTRIBUTION.size() >& indices, const array<uint8_t, DISTRIBUTION.size() >& change){
for(int i = 0; i < DISTRIBUTION.size() ; ++i){
assigned_days[indices[i]] = change[i];
}
setup() ;
}
};
static std::atomic<bool> flag(false);
static Index global_index({});
bool time_exit_fn() {
return duration_cast<minutes>(high_resolution_clock::now() -START_TIME ).count() < END_TIME;
}
void init_data() {
ifstream in(".. /input/santa-workshop-tour-2019/family_data.csv");
assert(in && "family_data.csv");
string header;
int n,x;
char comma;
getline(in, header);
for(int j = 0; j < choices.size() ; ++j){
in >> x >> comma;
for(int i = 0; i < 10; ++i){
in >> x >> comma;
choices[j][i] = x-1;
}
in >> n;
n_people[j] = n;
}
array<int, 10> pc{0, 50, 50, 100, 200, 200, 300, 300, 400, 500};
array<int, 10> pn{0, 0, 9, 9, 9, 18, 18, 36, 36, 235};
for(int j = 0; j < PCOSTM.size() ; ++j)
 for(int i = 0; i < 10; ++i)
 PCOSTM[j][i] = pc[i] + pn[i] * n_people[j];
for(int i = 0; i < 176; ++i)
 for(int j = 0; j < 176; ++j)
 ACOSTM[i][j] = i * pow(i+125, 0.5 + abs(i-j)/ 50.0)/ 400.0;
}
array<uint8_t, 5000> read_submission(string filename){
ifstream in(filename);
assert(in && "submission.csv");
array<uint8_t, 5000> assigned_day{};
string header;
int id, x;
char comma;
getline(in, header);
for(int j = 0; j < choices.size() ; ++j){
in >> id >> comma >> x;
assigned_day[j] = x-1;
auto it = find(begin(choices[j]), end(choices[j]), assigned_day[j]);
if(it != end(choices[j]))
assigned_day[j] = distance(begin(choices[j]), it);
}
return assigned_day;
}
double calc(const array<uint8_t, 5000>& assigned_days, bool print=false){
int preference_cost = 0;
double accounting_penalty = 0.0;
array<uint16_t, 100> daily_occupancy{};
for(int j = 0; j < assigned_days.size() ; ++j){
preference_cost += PCOSTM[j][assigned_days[j]];
daily_occupancy[choices[j][assigned_days[j]]] += n_people[j];
}
for(auto occupancy : daily_occupancy)
 if(occupancy < MIN_OCCUPANCY)
 return 1e12*(MIN_OCCUPANCY-occupancy);
else if(occupancy > MAX_OCCUPANCY)
 return 1e12*(occupancy - MAX_OCCUPANCY);
for(int day = 0; day < 99; ++day)
 accounting_penalty += ACOSTM[daily_occupancy[day]-125][daily_occupancy[day+1]-125];
accounting_penalty += ACOSTM[daily_occupancy[99]-125][daily_occupancy[99]-125];
if(print){
cout << preference_cost << " " << accounting_penalty << " " << preference_cost+accounting_penalty << endl;
}
return preference_cost + accounting_penalty;
}
void save_sub(const array<uint8_t, 5000>& assigned_day){
ofstream out("submission.csv");
out << "family_id,assigned_day" << endl;
for(int i = 0; i < assigned_day.size() ; ++i)
 out << i << "," << choices[i][assigned_day[i]]+1 << endl;
}
const vector<array<uint8_t, DISTRIBUTION.size() >> changes = []() {
vector<array<uint8_t, DISTRIBUTION.size() >> arr;
array<uint8_t, DISTRIBUTION.size() > tmp{};
for(int i = 0; true; ++i){
arr.push_back(tmp);
tmp[0] += 1;
for(int j = 0; j < DISTRIBUTION.size() ; ++j)
 if(tmp[j] >= DISTRIBUTION[j]){
if(j >= DISTRIBUTION.size() -1)
 return arr;
tmp[j] = 0;
++tmp[j+1];
}
}
return arr;
}() ;
//template<class ExitFunction>
void stochastic_product_search(Index index){ // 15'360'000it/s 65ns/it 0.065µs/it
double best_local_score = calc(index.assigned_days);
thread_local std::mt19937 gen(std::random_device{}());
uniform_int_distribution<> dis(0, 4999);
array<uint16_t, 5000> indices;
iota(begin(indices), end(indices), 0);
array<uint16_t, DISTRIBUTION.size() > best_indices{};
array<uint8_t, DISTRIBUTION.size() > best_change{};
for(;time_exit_fn() ;){
bool found_better = false;
for(int k = 0; k < BEST_N; ++k){
for(int i = 0; i < DISTRIBUTION.size() ; ++i)//random swap
swap(indices[i], indices[dis(gen)]);
for(const auto& change : changes){
auto score = index.calc(indices, change);
if(score < best_local_score){
found_better = true;
best_local_score = score;
best_change = change;
copy_n(begin(indices), DISTRIBUTION.size() , begin(best_indices)) ;
}
}
}
if(flag.load() == true){
return;
}
if(found_better && flag.load() == false){ // reindex from N best if found better
flag = true;
index.reindex(best_indices, best_change);
global_index = index;
return;
}
}
}
int main() {
init_data() ;
auto assigned_day = read_submission(".. /input/submission-710934/submission.csv");
Index index(assigned_day);
global_index = index;
calc(index.assigned_days, true);
for(;time_exit_fn() ;){
std::thread threads[N_JOBS];
for(int i = 0; i < N_JOBS; i++){
threads[i] = std::thread(stochastic_product_search, index);
}
for(int i = 0; i < N_JOBS; i++){
threads[i].join() ;
}
auto best_score = calc(global_index.assigned_days, true);
save_sub(global_index.assigned_days);
flag = false;
index = global_index;
}
return 0;
} 
<set_options>
def load_and_prec() :
train_df = pd.read_csv(".. /input/quora-insincere-questions-classification/train.csv")
 test_df = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv")
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: x.lower())
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, tokenizer.word_index
!g++ -pthread -lpthread -O3 -std=c++17 -o main main.cpp<define_variables>
tqdm.pandas()
if not os.path.exists('./train_X.npy'):
start_time = time.time()
train_X, test_X, train_y, word_index = load_and_prec()
total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))
np.save('./train_X', train_X, allow_pickle=True)
 np.save('./test_X', test_X, allow_pickle=True)
 np.save('./train_y', train_y, allow_pickle=True)
 with open('./word_index.pickle', 'wb')as handle:
pickle.dump(word_index, handle, protocol=pickle.HIGHEST_PROTOCOL)
 else :
train_X = np.load('./train_X.npy', allow_pickle=True)
 test_X = np.load('./test_X.npy', allow_pickle=True)
 train_y = np.load('./train_y.npy', allow_pickle=True)
 with open('./word_index.pickle', 'rb')as handle:
tokenizer = pickle.load(handle)
 gc.collect()
NMB_DAYS = 100
NMB_FAMILIES = 5000
MAX_PEOPLE_PERDAY = 300
MIN_PEOPLE_PERDAY = 125
MAX_PREF_ALLOWED = 4
cost_ = 0
assignment_ = [-1 for i in range(NMB_FAMILIES)]
nmb_people_assigned_to_day_ = [0 for i in range(NMB_DAYS)]
families_assigned_to_day_ = []
for i in range(NMB_DAYS):
families_assigned_to_day_.append([])
 def get_penalty(n, choice):
penalty = None
if choice == 0:
penalty = 0
elif choice == 1:
penalty = 50
elif choice == 2:
penalty = 50 + 9 * n
elif choice == 3:
penalty = 100 + 9 * n
elif choice == 4:
penalty = 200 + 9 * n
elif choice == 5:
penalty = 200 + 18 * n
elif choice == 6:
penalty = 300 + 18 * n
elif choice == 7:
penalty = 300 + 36 * n
elif choice == 8:
penalty = 400 + 36 * n
elif choice == 9:
penalty = 500 + 36 * n + 199 * n
else:
penalty = 500 + 36 * n + 398 * n
return penalty
def GetAssignmentCostMatrix(data):
cost_matrix = np.zeros(( NMB_FAMILIES, NMB_DAYS), dtype=np.int64)
 for i in range(NMB_FAMILIES):
desired = data.values[i, :-1]
cost_matrix[i, :] = get_penalty(FAMILY_SIZE[i], 10)
 for j, day in enumerate(desired):
cost_matrix[i, day-1] = get_penalty(FAMILY_SIZE[i], j)
 return cost_matrix
def GetAccountingCostMatrix() :
ac = np.zeros(( 1000, 1000), dtype=np.float64)
 for n in range(ac.shape[0]):
for n_p1 in range(ac.shape[1]):
diff = abs(n - n_p1)
 ac[n, n_p1] = max(0,(n - 125)/ 400.0 * n**(0.5 + diff / 50.0))
return ac
def GetPreferenceMatrix(data):
pref_matrix = np.zeros(( NMB_FAMILIES, NMB_DAYS), dtype=np.int64)
 for i in range(NMB_FAMILIES):
desired = data.values[i, :-1]
pref_matrix[i, :] = 10
for j, day in enumerate(desired):
pref_matrix[i, day-1] = j
return pref_matrix
def GetPreferenceForFamiliesMatrix(data):
pref_matrix = np.zeros(( NMB_FAMILIES, NMB_DAYS), dtype=np.int64)
 pref_matrix2 = np.zeros(( NMB_FAMILIES, 10), dtype=np.int64)
 for i in range(NMB_FAMILIES):
desired = data.values[i, :-1]
pref_matrix[i, :] = 10
for j, day in enumerate(desired):
pref_matrix[i, day-1] = j
for i in range(NMB_FAMILIES):
for j in range(NMB_DAYS):
if(pref_matrix[i][j] < 10):
pref_matrix2[i][pref_matrix[i][j]] = j
return pref_matrix2
swap_candidates_ = []
for i in range(NMB_DAYS):
swap_candidates_.append([])
 for j in range(NMB_DAYS):
swap_candidates_[i].append([])
 def preprocessing() :
nbTopForBinN = {}
for i in range(10):
for j in range(NMB_DAYS):
nbTopForBinN[i, j] = 0
for F in range(NMB_FAMILIES):
for i in range(MAX_PREF_ALLOWED + 1):
D = PREFERENCES_FOR_FAMILY[F][i]
for p in range(i, MAX_PREF_ALLOWED + 1):
nbTopForBinN[p, D] += FAMILY_SIZE[F]
nmbVarsDROPPED = 0
for i in range(NMB_DAYS):
if nbTopForBinN[0, i] < 100:
for F in range(NMB_FAMILIES):
if PREFERENCES_FOR_FAMILY[F][0] == i:
for P in range(1, MAX_PREF_ALLOWED + 1):
if PREFERENCE_MATRIX[F, PREFERENCES_FOR_FAMILY[F][P]] < 10:
nmbVarsDROPPED += 1
PREFERENCE_MATRIX[F, PREFERENCES_FOR_FAMILY[F][P]] = 10
for D in range(NMB_DAYS):
if nbTopForBinN[0, D] >= 300:
for F in range(NMB_FAMILIES):
for P in range(2, MAX_PREF_ALLOWED + 1):
if PREFERENCES_FOR_FAMILY[F][P] == D:
if PREFERENCE_MATRIX[F, PREFERENCES_FOR_FAMILY[F][P]] < 10:
nmbVarsDROPPED += 1
PREFERENCE_MATRIX[F, PREFERENCES_FOR_FAMILY[F][P]] = 10
for D in range(NMB_DAYS):
if nbTopForBinN[1, D] >= 300 and nbTopForBinN[0, D] < 300:
for F in range(NMB_FAMILIES):
for P in range(3, MAX_PREF_ALLOWED + 1):
if PREFERENCES_FOR_FAMILY[F][P] == D:
if PREFERENCE_MATRIX[F, PREFERENCES_FOR_FAMILY[F][P]] < 10:
nmbVarsDROPPED += 1
PREFERENCE_MATRIX[F, PREFERENCES_FOR_FAMILY[F][P]] = 10
for D in range(NMB_DAYS):
if nbTopForBinN[2, D] >= 300 and nbTopForBinN[0, D] < 300 and nbTopForBinN[1, D] < 300:
for F in range(NMB_FAMILIES):
for P in range(4, MAX_PREF_ALLOWED + 1):
if PREFERENCES_FOR_FAMILY[F][P] == D:
if PREFERENCE_MATRIX[F, PREFERENCES_FOR_FAMILY[F][P]] < 10:
nmbVarsDROPPED += 1
PREFERENCE_MATRIX[F, PREFERENCES_FOR_FAMILY[F][P]] = 10
def calculate_solution_cost() :
global cost_
cost_ = 0
assign_cost = 0
account_cost = 0
for i in range(NMB_FAMILIES):
cost_ += ASSIGNMENT_COST_MATRIX[i][assignment_[i]]
assign_cost = cost_
for d in range(NMB_DAYS):
today_count = nmb_people_assigned_to_day_[d]
yesterday_count = today_count
if(d < NMB_DAYS - 1):
yesterday_count = nmb_people_assigned_to_day_[d + 1]
cost_ += ACCOUNTING_COST_MATRIX[today_count][yesterday_count]
account_cost += ACCOUNTING_COST_MATRIX[today_count][yesterday_count]
return cost_, assign_cost, account_cost
def update_solution(assign):
global cost_
global nmb_people_assigned_to_day_
global assignment_
for i in range(NMB_DAYS):
nmb_people_assigned_to_day_[i] = 0
families_assigned_to_day_[i] = []
for j in range(NMB_DAYS):
swap_candidates_[i][j] = []
for F in range(NMB_FAMILIES):
day = assign[F]
assignment_[F] = day
nmb_people_assigned_to_day_[day] += FAMILY_SIZE[F]
families_assigned_to_day_[day].append(F)
 for k in range(MAX_PREF_ALLOWED + 1):
DD = PREFERENCES_FOR_FAMILY[F][k]
if(PREFERENCE_MATRIX[F][DD] <= MAX_PREF_ALLOWED):
swap_candidates_[day][DD].append(F);
calculate_solution_cost()
def calculate_accounting_cost(nmb_people_assigned_to_day_, days_to_calc):
accounting_cost = 0
for d in days_to_calc:
today_count = nmb_people_assigned_to_day_[d]
yesterday_count = nmb_people_assigned_to_day_[d]
if(d < NMB_DAYS - 1):
yesterday_count = nmb_people_assigned_to_day_[d + 1]
accounting_cost += ACCOUNTING_COST_MATRIX[today_count][yesterday_count]
return accounting_cost
def check_move(F1, F2, D2):
global cost_
global nmb_people_assigned_to_day_
global assignment_
D1 = assignment_[F1]
N1 = FAMILY_SIZE[F1]
N2 = 0
if(F2 >= 0):
N2 = FAMILY_SIZE[F2]
if(nmb_people_assigned_to_day_[D1] - N1 + N2 < MIN_PEOPLE_PERDAY):
return False
if(nmb_people_assigned_to_day_[D1] - N1 + N2 > MAX_PEOPLE_PERDAY):
return False
if(nmb_people_assigned_to_day_[D2] - N2 + N1 < MIN_PEOPLE_PERDAY):
return False
if(nmb_people_assigned_to_day_[D2] - N2 + N1 > MAX_PEOPLE_PERDAY):
return False
return True
def calculate_assignment_cost_diff_with_move(F1, F2, D2):
D1 = assignment_[F1]
cost_diff = 0
cost_diff +=(ASSIGNMENT_COST_MATRIX[F1][D2] - ASSIGNMENT_COST_MATRIX[F1][D1])
 if(F2 >= 0):
cost_diff +=(ASSIGNMENT_COST_MATRIX[F2][D1] - ASSIGNMENT_COST_MATRIX[F2][D2])
 return cost_diff
def calculate_accounting_cost_diff_with_move(F1, F2, D2):
D1 = assignment_[F1]
N1 = FAMILY_SIZE[F1]
N2 = 0
if(F2 >= 0):
N2 = FAMILY_SIZE[F2]
accounting_cost_diff = 0
accounting_cost_old = 0
accounting_cost_new = 0
days_to_calc = {D1, D2}
if(D1 > 0):
days_to_calc.add(D1 - 1)
 if(D2 > 0):
days_to_calc.add(D2 - 1)
 accounting_cost_old = calculate_accounting_cost(nmb_people_assigned_to_day_, days_to_calc)
 nmb_people_assigned_to_day_new_ = list(nmb_people_assigned_to_day_)
 nmb_people_assigned_to_day_new_[D1] +=(N2 - N1)
 nmb_people_assigned_to_day_new_[D2] +=(N1 - N2)
 accounting_cost_new = calculate_accounting_cost(nmb_people_assigned_to_day_new_, days_to_calc)
 accounting_cost_diff = accounting_cost_new - accounting_cost_old
return accounting_cost_diff
def perform_move(F1, F2, D2):
global cost_
global nmb_people_assigned_to_day_
global assignment_
D1 = assignment_[F1]
N1 = FAMILY_SIZE[F1]
N2 = 0
if(F2 >= 0):
N2 = FAMILY_SIZE[F2]
assignment_[F1] = D2
families_assigned_to_day_[D2].append(F1)
 families_assigned_to_day_[D1].remove(F1)
 for k in range(MAX_PREF_ALLOWED + 1):
DD = PREFERENCES_FOR_FAMILY[F1][k]
if(PREFERENCE_MATRIX[F1][DD] <= MAX_PREF_ALLOWED):
swap_candidates_[D2][DD].append(F1);
swap_candidates_[D1][DD].remove(F1);
if(F2 >= 0):
assignment_[F2] = D1
families_assigned_to_day_[D1].append(F2)
 families_assigned_to_day_[D2].remove(F2)
 for k in range(MAX_PREF_ALLOWED + 1):
DD = PREFERENCES_FOR_FAMILY[F2][k]
if(PREFERENCE_MATRIX[F2][DD] <= MAX_PREF_ALLOWED):
swap_candidates_[D1][DD].append(F2);
swap_candidates_[D2][DD].remove(F2);
nmb_people_assigned_to_day_[D1] +=(N2 - N1)
 nmb_people_assigned_to_day_[D2] +=(N1 - N2)
 calculate_solution_cost()
TOLRAND = 1000
TOL = 50
assCostDiffTOL = 10000000
def local_search_shift_and_swap(nmbIters):
iter = 0
while iter < nmbIters:
iter+=1
F1 = random.randint(0, NMB_FAMILIES - 1)
 D1 = assignment_[F1]
r = random.randint(0, MAX_PREF_ALLOWED)
 D2 = PREFERENCES_FOR_FAMILY[F1][r]
if(D1 == D2):
continue
if(PREFERENCE_MATRIX[F1][D2] > MAX_PREF_ALLOWED):
continue
F2 = -1
if(random.randint(0, 100)< 10):
if len(swap_candidates_[D2][D1])== 0:
continue
r = random.randint(0, len(swap_candidates_[D2][D1])- 1)
 F2 = swap_candidates_[D2][D1][r]
if(PREFERENCE_MATRIX[F2][D1] > MAX_PREF_ALLOWED):
continue
if(check_move(F1, F2, D2)== False):
continue
ass_cost_diff = calculate_assignment_cost_diff_with_move(F1, F2, D2)
 if(ass_cost_diff > assCostDiffTOL):
continue
acc_cost_diff = calculate_accounting_cost_diff_with_move(F1, F2, D2)
 cost_diff = ass_cost_diff + acc_cost_diff
tol = 0
if(random.randint(0, TOLRAND)== 1):
tol = TOL
if(cost_diff <= tol and ass_cost_diff <= assCostDiffTOL):
perform_move(F1, F2, D2)
 def LocalSearch(timeLimit, maxIters):
startTime = time.time()
iter = 0
global assCostDiffTOL
assCostDiffTOL = 0
best_assignment = {}
for i in range(NMB_FAMILIES):
best_assignment[i] = assignment_[i]
bestCost = cost_
while(iter < maxIters):
if(time.time() - startTime >= timeLimit):
break
iter+=1
if(iter % 5 == 0): 	
update_solution(best_assignment)
 local_search_shift_and_swap(20000)
 if(cost_ < bestCost):
bestCost = cost_
print(iter, bestCost, int(time.time() - startTime))
for i in range(NMB_FAMILIES):
best_assignment[i] = assignment_[i]
assCostDiffTOL += 1
update_solution(best_assignment)
 def reg_m(y, x):
ones = np.ones(len(x[0]))
X = sm.add_constant(np.column_stack(( x[0], ones)))
 for ele in x[1:]:
X = sm.add_constant(np.column_stack(( ele, X)))
 results = sm.OLS(y, X ).fit()
return results
DELTA = 1
LINEARIZE = False
def linearize() :
resultABC = []
for d in range(NMB_DAYS):
resultABC.append([])
 for D in range(NMB_DAYS):
x = []
x.append([])
 x.append([])
 y = []
LB = max(MIN_PEOPLE_PERDAY, nmb_people_assigned_to_day_[D] - DELTA)
 UB = min(MAX_PEOPLE_PERDAY, nmb_people_assigned_to_day_[D] + DELTA)
 dom1 = {LB}
for i in range(LB + 1, UB + 1):
dom1.add(i)
 if D < NMB_DAYS - 1:
LB = max(MIN_PEOPLE_PERDAY, nmb_people_assigned_to_day_[D + 1] - DELTA)
 UB = min(MAX_PEOPLE_PERDAY, nmb_people_assigned_to_day_[D + 1] + DELTA)
 dom2 = {LB}
for i in range(LB + 1, UB + 1):
dom2.add(i)
 for i in dom1:
for j in dom2:
x[0].append(i)
 x[1].append(j)
 y.append(ACCOUNTING_COST_MATRIX[i, j])
 else:
for i in dom1:
x[0].append(i)
 x[1].append(i)
 y.append(ACCOUNTING_COST_MATRIX[i, i])
 result = reg_m(y,x)
 if(nmb_people_assigned_to_day_[D] == 125):
result.params[0] = 0
result.params[1] = 10000
result.params[2] = -125 * 10000
resultABC[D] = result.params
p = resultABC[D][0]
resultABC[D][0] = resultABC[D][1]
resultABC[D][1] = p
return resultABC
def MIP(timeLimit):
solver = pywraplp.Solver('simple_mip_program', pywraplp.Solver.CBC_MIXED_INTEGER_PROGRAMMING)
 x = {}
C = {}
key = {}
K = 0
for i in range(NMB_FAMILIES):
for j in range(NMB_DAYS):
if PREFERENCE_MATRIX[i, j] <= MAX_PREF_ALLOWED:
key[i, j] = K
x[K] = solver.BoolVar('x[%i,%i]' %(i, j))
C[K] = ASSIGNMENT_COST_MATRIX[i, j]
K = K + 1
else:
key[i, j] = -1
nmbvar = solver.NumVariables()
print('Number of variables =', solver.NumVariables())
 for i in range(NMB_FAMILIES):
expr = 0
for j in range(NMB_DAYS):
if key[i, j] >= 0:
expr = expr + x[key[i, j]]
solver.Add(expr == 1)
 occ = {}
for i in range(NMB_DAYS):
occ[i] = solver.IntVar(0.0, 1000.0, 'occ[%i,%i]' %(i, 0))
for i in range(NMB_DAYS):
expr = 0
for j in range(NMB_FAMILIES):
if key[j, i] >= 0:
expr +=(x[key[j, i]] * FAMILY_SIZE[j])
 LB = MIN_PEOPLE_PERDAY
UB = MAX_PEOPLE_PERDAY
if LINEARIZE:
LB = max(nmb_people_assigned_to_day_[i] - DELTA, MIN_PEOPLE_PERDAY)
 UB = min(nmb_people_assigned_to_day_[i] + DELTA, MAX_PEOPLE_PERDAY)
 solver.Add(expr >= LB)
 solver.Add(expr <= UB)
 solver.Add(occ[i] == expr)
 print('Number of constraints =', solver.NumConstraints())
 obj = solver.Sum([C[i] * x[i] for i in range(nmbvar)])
 if LINEARIZE == True:
obj2 = 0
ABC = linearize()
for D in range(NMB_DAYS - 1):
a = ABC[D][0]
b = ABC[D][1]
c = ABC[D][2]
if D < NMB_DAYS - 1:
obj2 = obj2 +(a * occ[D] + b * occ[D + 1] + c)
 else:
obj2 = obj2 +(a * occ[D] + b * occ[D] + c)
 obj = obj + 1 * obj2
solver.Minimize(obj)
 solver.SetTimeLimit(1000 * timeLimit)
 status = solver.Solve()
print('MIP Objective value =', solver.Objective().Value())
 print('Problem solved in %f milliseconds' % solver.wall_time())
 assign = {}
for i in range(NMB_FAMILIES):
for j in range(NMB_DAYS):
if key[i, j] >= 0:
if x[key[i, j]].solution_value() > 0.99:
assign[i] = j
update_solution(assign)
 random.seed(52)
 data = pd.read_csv('/kaggle/input/santa-workshop-tour-2019/family_data.csv', index_col='family_id')
 FAMILY_SIZE = data.n_people.values
ASSIGNMENT_COST_MATRIX = GetAssignmentCostMatrix(data)
 ACCOUNTING_COST_MATRIX = GetAccountingCostMatrix()
PREFERENCE_MATRIX = GetPreferenceMatrix(data)
 PREFERENCES_FOR_FAMILY = GetPreferenceForFamiliesMatrix(data)
 preprocessing()
startTime = time.time()
MIP(60)
 print(calculate_solution_cost())
 LocalSearch(3600, 2000)
 print(calculate_solution_cost())
 DELTA = 1
LINEARIZE = True
MIP(5 * 60)
 print(calculate_solution_cost())
 LocalSearch(10 * 3600, 500)
 best_assignment = {}
for i in range(NMB_FAMILIES):
best_assignment[i] = assignment_[i]
bestCost = cost_
for i in range(5):
MIP(5 * 60)
 print(calculate_solution_cost())
 LocalSearch(3600, 100)
 print(calculate_solution_cost())
 if(cost_ < bestCost):
bestCost = cost_
print("bestCost: ", bestCost, " time: ", time.time() - startTime)
 for i in range(NMB_FAMILIES):
best_assignment[i] = assignment_[i]
update_solution(best_assignment)
 print("Best Solution: ", bestCost)
 print("LocalSearch...")
 LocalSearch(3600, 1000)
 print("Final Solution: ", cost_)
 with open('submission.csv', mode='w')as csv_file:
fieldnames = ['family_id', 'assigned_day']
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
 writer.writeheader()
for i in range(NMB_FAMILIES):
writer.writerow({'family_id': i, 'assigned_day': assignment_[i] + 1} )<load_from_csv>
def load_glove(word_index):
EMBEDDING_FILE = '.. /working/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index)+1)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
%matplotlib inline
NUMBER_DAYS = 100
NUMBER_FAMILIES = 5000
data = pd.read_csv('/kaggle/input/santa-workshop-tour-2019/family_data.csv')
 submission = pd.read_csv('/kaggle/input/c-stochastic-product-search-65ns/submission.csv')
 assigned_days = submission['assigned_day'].values
columns = data.columns[1:11]
DESIRED = data[columns].values
COST_PER_FAMILY = [0,50,50,100,200,200,300,300,400,500]
COST_PER_FAMILY_MEMBER = [0, 0, 9, 9, 9, 18, 18, 36, 36,235]
N_PEOPLE = data['n_people'].values
def get_daily_occupancy(assigned_days):
daily_occupancy = np.zeros(100, int)
 for fid, assigned_day in enumerate(assigned_days):
daily_occupancy[assigned_day-1] += N_PEOPLE[fid]
return daily_occupancy
def cost_function(prediction):
N_DAYS = 100
MAX_OCCUPANCY = 300
MIN_OCCUPANCY = 125
penalty = 0
days = list(range(N_DAYS,0,-1))
tmp = pd.read_csv('/kaggle/input/santa-workshop-tour-2019/family_data.csv', index_col='family_id')
 family_size_dict = tmp[['n_people']].to_dict() ['n_people']
cols = [f'choice_{i}' for i in range(10)]
choice_dict = tmp[cols].to_dict()
daily_occupancy = {k:0 for k in days}
for f, d in enumerate(prediction):
n = family_size_dict[f]
choice_0 = choice_dict['choice_0'][f]
choice_1 = choice_dict['choice_1'][f]
choice_2 = choice_dict['choice_2'][f]
choice_3 = choice_dict['choice_3'][f]
choice_4 = choice_dict['choice_4'][f]
choice_5 = choice_dict['choice_5'][f]
choice_6 = choice_dict['choice_6'][f]
choice_7 = choice_dict['choice_7'][f]
choice_8 = choice_dict['choice_8'][f]
choice_9 = choice_dict['choice_9'][f]
daily_occupancy[d] += n
if d == choice_0:
penalty += 0
elif d == choice_1:
penalty += 50
elif d == choice_2:
penalty += 50 + 9 * n
elif d == choice_3:
penalty += 100 + 9 * n
elif d == choice_4:
penalty += 200 + 9 * n
elif d == choice_5:
penalty += 200 + 18 * n
elif d == choice_6:
penalty += 300 + 18 * n
elif d == choice_7:
penalty += 300 + 36 * n
elif d == choice_8:
penalty += 400 + 36 * n
elif d == choice_9:
penalty += 500 + 36 * n + 199 * n
else:
penalty += 500 + 36 * n + 398 * n
for _, v in daily_occupancy.items() :
if(v < MIN_OCCUPANCY):
penalty += 100000000
accounting_cost =(daily_occupancy[days[0]]-125.0)/ 400.0 * daily_occupancy[days[0]]**(0.5)
 accounting_costs = [max(0, accounting_cost)]
diffs = [0]
yesterday_count = daily_occupancy[days[0]]
for day in days[1:]:
today_count = daily_occupancy[day]
diff = abs(today_count - yesterday_count)
 accounting_costs.append(max(0,(today_count-125.0)/ 400.0 * today_count**(0.5 + diff / 50.0)))
 yesterday_count = today_count
return penalty, sum(accounting_costs), penalty + sum(accounting_costs )<define_variables>
if not os.path.exists('./embedding_matrix.npy'):
start_time = time.time()
embedding_matrix = load_glove(word_index)
 print(np.shape(embedding_matrix))
total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))
np.save('./embedding_matrix', embedding_matrix, allow_pickle=True)
 else :
embedding_matrix = np.load('./embedding_matrix.npy', allow_pickle=True)
 gc.collect()
MAX_BEST_CHOICE = 5
NUM_SWAP = 2500
NUM_SECONDS = 1800
NUM_THREADS = 4
for _ in range(40):
solver = pywraplp.Solver('Optimization preference cost', pywraplp.Solver.CBC_MIXED_INTEGER_PROGRAMMING)
 daily_occupancy = get_daily_occupancy(assigned_days ).astype(float)
 fids = np.random.choice(range(NUMBER_FAMILIES), NUM_SWAP, replace=False)
 PCOSTM, B = {}, {}
for fid in range(NUMBER_FAMILIES):
if fid in fids:
for i in range(MAX_BEST_CHOICE):
PCOSTM[fid, DESIRED[fid][i]-1] = COST_PER_FAMILY[i] + N_PEOPLE[fid] * COST_PER_FAMILY_MEMBER[i]
B[ fid, DESIRED[fid][i]-1] = solver.BoolVar('')
 else:
daily_occupancy[assigned_days[fid]-1] -= N_PEOPLE[fid]
solver.set_time_limit(NUM_SECONDS*NUM_THREADS*1000)
 solver.SetNumThreads(NUM_THREADS)
 for day in range(NUMBER_DAYS):
if daily_occupancy[day]:
solver.Add(solver.Sum([N_PEOPLE[fid] * B[fid, day] for fid in range(NUMBER_FAMILIES)if(fid,day)in B])== daily_occupancy[day])
 for fid in fids:
solver.Add(solver.Sum(B[fid, day] for day in range(NUMBER_DAYS)if(fid, day)in B)== 1)
 solver.Minimize(solver.Sum(PCOSTM[fid, day] * B[fid, day] for fid, day in B))
sol = solver.Solve()
status = ['OPTIMAL', 'FEASIBLE', 'INFEASIBLE', 'UNBOUNDED', 'ABNORMAL', 'MODEL_INVALID', 'NOT_SOLVED']
if status[sol] in ['OPTIMAL', 'FEASIBLE']:
tmp = assigned_days.copy()
for fid, day in B:
if B[fid, day].solution_value() > 0.5:
tmp[fid] = day+1
if cost_function(tmp)[2] < cost_function(assigned_days)[2]:
assigned_days = tmp
submission['assigned_day'] = assigned_days
submission.to_csv('submission.csv', index=False)
 print('Result:', status[sol], cost_function(tmp))
else:
print('Result:', status[sol] )<import_modules>
class BasicNet(nn.Module):
def __init__(self):
super(BasicNet, self ).__init__()
hidden_size = 60
self.word_layer = 16
num_layers = 2
dropout = 0.1
self.embedding = nn.Embedding(max_features, embed_size, padding_idx=0)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.lstm = nn.LSTM(input_size=embed_size, hidden_size=hidden_size, num_layers=num_layers, dropout=dropout, bidirectional=True, batch_first=True)
 self.sigmoid = nn.Sigmoid()
self.linear_word = nn.Linear(hidden_size*2, self.word_layer)
 self.linear_sentence = nn.Linear(self.word_layer*maxlen, 1)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_lstm, _ = self.lstm(h_embedding)
 h_lstm = self.sigmoid(h_lstm)
 result = self.sigmoid(self.linear_word(h_lstm))
result = self.dropout(result)
 result = result.reshape(( len(result),-1,))
result = self.linear_sentence(result)
 return result
import os
import ctypes
from numpy.ctypeslib import ndpointer
import pandas as pd
import numpy as np
from tqdm.notebook import tqdm
from numba import njit, prange<feature_engineering>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
%%writefile score.c
int cost[NF][101];
int fs[NF];
int cf[NF][10];
int loaded=0;
float acc[301][301];
void precompute_acc() {
for(int i=125;i<=300;i++)
 for(int j=125;j<=300;j++)
 acc[i][j] =(i-125.0)/400.0 * pow(i , 0.5 + fabs(i-j)/ 50);
}
void read_fam() {
FILE *f;
char s[1000];
int d[101],fid,n;
int *c;
f=fopen(".. /input/santa-workshop-tour-2019/family_data.csv","r");
if(fgets(s,1000,f)==NULL)
 exit(-1);
for(int i=0;i<5000;i++){
c = &cf[i][0];
if(fscanf(f,"%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d",
&fid,&c[0],&c[1],&c[2],&c[3],&c[4],&c[5],&c[6],&c[7],&c[8],&c[9],&fs[i])!=12)
 exit(-1);
// printf("%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d
",
//fid,c[0],c[1],c[2],c[3],c[4],c[5],c[6],c[7],c[8],c[9],fs[i]);
n = fs[i];
for(int j=1;j<=100;j++){
if(j==c[0])cost[i][j]=0;
else if(j==c[1])cost[i][j]=50;
else if(j==c[2])cost[i][j]=50 + 9 * n;
else if(j==c[3])cost[i][j]=100 + 9 * n;
else if(j==c[4])cost[i][j]=200 + 9 * n;
else if(j==c[5])cost[i][j]=200 + 18 * n;
else if(j==c[6])cost[i][j]=300 + 18 * n;
else if(j==c[7])cost[i][j]=300 + 36 * n;
else if(j==c[8])cost[i][j]=400 + 36 * n;
else if(j==c[9])cost[i][j]=500 + 36 * n + 199 * n;
else cost[i][j]=500 + 36 * n + 398 * n;
}
}
}
float max_cost=1000000000;
int day_occ[102];
static inline int day_occ_ok(int d){
return !(d <125 || d>300);
}
float score(int *pred){
float r=0;
if(!loaded){
read_fam() ;
precompute_acc() ;
loaded = 1;
}
// validate day occupancy
memset(day_occ,0,101*sizeof(int)) ;
for(int i=0;i<NF;i++){
day_occ[pred[i]]+=fs[i];
r+=cost[i][pred[i]];
}
day_occ[101]=day_occ[100];
for(int d=1;d<=100;d++){
if(day_occ[d]<125)
 r += 100000 *(125 - day_occ[d]);
else if(day_occ[d] > 300)
 r += 100000 *(day_occ[d] - 300);
r += acc[day_occ[d]][day_occ[d+1]];
}
return r;
}<data_type_conversions>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
lib = ctypes.CDLL('./score.so')
 cost_function = lib.score
cost_function.restype = ctypes.c_float
cost_function.argtypes = [ndpointer(ctypes.c_int)]<load_from_csv>
train_epochs = 5
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED ).split(train_X, train_y))
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 seed_torch(SEED)
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = BasicNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(weight=None, reduction="sum")
 optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))
for j,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[j * batch_size:(j+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )
score = []
sub = []
name = os.listdir('/kaggle/input/santa-public')
 for item in name:
score.append(int(item.split('_')[1].split('.')[0]))
sub.append(pd.read_csv('.. /input/santa-public/'+item, index_col='family_id'))
print(np.min(score))
print(len(sub))<load_from_csv>
search_result = threshold_search(train_y, train_preds)
 search_result
<define_variables><EOS> 
sub = pd.read_csv('.. /input/quora-insincere-questions-classification/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub['prediction'] = sub['prediction'].apply(lambda x : int(x))
sub.to_csv("./submission.csv", index=False )