Kaggle competition: House Prices - Advanced Regression Techniques
gpt2_base_models = ['gpt2_epoch_1_lb_093912.bin', 'gpt2_epoch_1_lb_093904.bin', 'gpt2_k_epoch_2_lb_093902.bin']<categorify>
warnings.filterwarnings('ignore' )
x_test = gpt2_convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), 220, gpt2_tokenizer )<load_pretrained>
train_data = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv' )
gpt2_base_preds = np.zeros(( x_test.shape[0], len(gpt2_base_models)))
 for i, gpt2_model in enumerate(gpt2_base_models):
model = GPT2ClassificationHeadModel.from_pretrained('.. /input/submission-toxicity-classification/gpt2-models/')
 model.load_state_dict(torch.load(os.path.join(MODELS_PATH, gpt2_model)))
 model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
preds = gpt2_get_preds(model, x_test)
 gpt2_base_preds[:,i] = preds
del model
gc.collect()<prepare_output>
test_data = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
test_pred =(np.average(bert_base_preds, axis=1)+ np.average(gpt2_base_preds, axis=1)) / 2<save_to_csv>
obj_list = []
for i in train_data.columns :
if train_data[i].dtypes == 'object':
obj_list.append(i )
submission = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': test_pred
})
 submission.to_csv('submission.csv', index=False )<choose_model_class>
train_data.drop(['PoolQC', 'Utilities', 'Street', 'MiscFeature'], axis=1, inplace=True)
 test_data.drop(['PoolQC', 'Utilities', 'Street', 'MiscFeature'], axis=1, inplace=True )
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english")
<categorify>
train_data['PoolStatus'] = train_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 train_data['SeFlrStatus'] = train_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 train_data['GarageStatus'] = train_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 train_data['BsmtStatus'] = train_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 train_data['FirePlaceStatus'] = train_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 test_data['PoolStatus'] = test_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 test_data['SeFlrStatus'] = test_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 test_data['GarageStatus'] = test_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 test_data['BsmtStatus'] = test_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 test_data['FirePlaceStatus'] = test_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 train_data['RemodNBuild']=train_data['YearBuilt']+train_data['YearRemodAdd']
train_data['SF']=train_data['TotalBsmtSF'] + train_data['1stFlrSF'] + train_data['2ndFlrSF']
train_data['AreaInFt'] =(train_data['BsmtFinSF1'] + train_data['BsmtFinSF2'] +
train_data['1stFlrSF'] + train_data['2ndFlrSF'])
 train_data['NumBathroom'] =(train_data['FullBath'] +(0.5 * train_data['HalfBath'])+
train_data['BsmtFullBath'] +(0.5 * train_data['BsmtHalfBath']))
train_data['Total_porch_sf'] =(train_data['OpenPorchSF'] + train_data['3SsnPorch'] +
train_data['EnclosedPorch'] + train_data['ScreenPorch'] +
train_data['WoodDeckSF'])
 test_data['RemodNBuild']=test_data['YearBuilt']+test_data['YearRemodAdd']
test_data['SF']=test_data['TotalBsmtSF'] + test_data['1stFlrSF'] + test_data['2ndFlrSF']
test_data['AreaInFt'] =(test_data['BsmtFinSF1'] + test_data['BsmtFinSF2'] +
test_data['1stFlrSF'] + test_data['2ndFlrSF'])
 test_data['NumBathroom'] =(test_data['FullBath'] +(0.5 * test_data['HalfBath'])+
test_data['BsmtFullBath'] +(0.5 * test_data['BsmtHalfBath']))
test_data['Total_porch_sf'] =(test_data['OpenPorchSF'] + test_data['3SsnPorch'] +
test_data['EnclosedPorch'] + test_data['ScreenPorch'] +
test_data['WoodDeckSF'] )
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`."
return(e2 for e1 in edits1(word)for e2 in edits1(e1))
def singlify(word):
return "".join([letter for i,letter in enumerate(word)if i == 0 or letter != word[i-1]] )<features_selection>
ID = test_data['Id']
del test_data['Id']
def load_glove(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
def load_fasttext(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words 
<choose_model_class>
del train_data['Id']
def build_model_simplelstm(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.3 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam' , metrics=['accuracy'])
 return model
def build_model2(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.3 )(x)
 x1 = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x1),
GlobalMaxPooling1D()(x2),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam' , metrics=['accuracy'])
 return model 
<load_from_csv>
from sklearn_pandas import DataFrameMapper, CategoricalImputer
from sklearn.pipeline import Pipeline, FeatureUnion
import xgboost as xgb
from sklearn.preprocessing import Imputer, FunctionTransformer, MinMaxScaler
from sklearn.impute import SimpleImputer
from sklearn.model_selection import cross_val_score
from sklearn.feature_extraction import DictVectorizer
start_time = time.time()
print("Loading data...")
 train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 print("--- %s seconds ---" %(time.time() - start_time))<set_options>
quasi_constant = []
for i in train_data.columns:
counts = train_data[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(train_data)* 100 > 99.94:
quasi_constant.append(i)
 train_data = train_data.drop(quasi_constant, axis=1)
 test_data = test_data.drop(quasi_constant, axis=1 )
gc.collect()<feature_engineering>
list_of_numerical = []
list_of_categorical = []
for i in train_data.columns:
if train_data[i].dtypes == 'object':
list_of_categorical.append(i)
 else:
if i == 'SalePrice':
pass
else:
list_of_numerical.append(i )
start_time = time.time()
print("Spacy NLP...")
 nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])
 nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
 word_dict = {}
word_index = 1
lemma_dict = {}
docs = nlp.pipe(pd.concat([train['comment_text'], test['comment_text']]), n_threads = 2)
 word_sequences = []
for doc in tqdm(docs):
word_seq = []
for token in doc:
if(token.text not in word_dict)and(token.pos_ is not "PUNCT"):
word_dict[token.text] = word_index
word_index += 1
lemma_dict[token.text] = token.lemma_
if token.pos_ is not "PUNCT":
word_seq.append(word_dict[token.text])
 word_sequences.append(word_seq)
 del docs
gc.collect()
num_train_data = len(train)
 train_word_sequences = word_sequences[:num_train_data]
test_word_sequences = word_sequences[num_train_data:]
print("--- %s seconds ---" %(time.time() - start_time))<set_options>
transformers = []
transformers.extend([([num_feat],SimpleImputer(strategy='constant',fill_value=-1)) for num_feat in list_of_numerical])
 transformers.extend([(cat_feat,CategoricalImputer())for cat_feat in list_of_categorical])
 combined_pipeline = DataFrameMapper(transformers,
input_df=True,
df_out=True )
del nlp
gc.collect()<load_pretrained>
def return_dict(blob):
return blob.to_dict("records" )
start_time = time.time()
print("Build spell correction dictnary...")
 spell_model = gensim.models.KeyedVectors.load_word2vec_format('.. /input/wikinews300d1mvec/wiki-news-300d-1M.vec')
 words = spell_model.index2word
w_rank = {}
for i,word in enumerate(words):
w_rank[word] = i
WORDS = w_rank
print("--- %s seconds ---" %(time.time() - start_time))<drop_column>
X_train, X_test, y_train, y_test = train_test_split(
 train_data.drop(['SalePrice'],axis=1), train_data['SalePrice'], test_size=0.15, random_state=42 )
del spell_model
del words
del w_rank
gc.collect()<train_model>
pipeline1 = Pipeline([ 
('featureunion',combined_pipeline), 
('dictifier',FunctionTransformer(func=return_dict,validate=False)) , 
('vectorizer',DictVectorizer(sort=False,sparse=False)) , 
('reg',xgb.XGBRegressor(objective="reg:squarederror"))
] )
start_time = time.time()
print("Loading embedding matrix...")
 embedding_matrix_glove, nb_words = load_glove(word_dict, lemma_dict)
 print("--- %s seconds ---" %(time.time() - start_time))<set_options>
mse1 = cross_val_score(pipeline1, X_train, y_train, scoring="neg_mean_squared_log_error", cv=5 )
del word_dict
del lemma_dict
gc.collect()<categorify>
print(np.sqrt(-mse1))
max_length = 220
embedding_size = 300
num_epoch = 4
train_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')
 test_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')
 print(train_word_sequences[:1])
 print(test_word_sequences[:1])
 pred_prob = np.zeros(( len(test_word_sequences),), dtype=np.float32 )<set_options>
from sklearn.model_selection import RandomizedSearchCV
gc.collect()<define_variables>
params = {
'reg__max_depth' : [5,10,15,20],
'reg__gamma' : [.1,.2,.3,.4,.5,.6],
'reg__colsample_bytree' : [.1,.2,.3,.4,.5,.6,.7],
'reg__n_estimators' : [50,100,200]
}
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train[column] = np.where(train[column] >= 0.5, True, False)
 sample_weights = np.ones(len(train), dtype=np.float32)
 sample_weights += np.array(train[IDENTITY_COLUMNS].sum(axis=1))
sample_weights += train[TARGET_COLUMN] *(~train[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train[TARGET_COLUMN])* train[IDENTITY_COLUMNS].sum(axis=1)* 5
sample_weights /= sample_weights.mean()<prepare_x_and_y>
grid_pipe_xgboost = RandomizedSearchCV(estimator=pipeline1,param_distributions=params,cv=4,
scoring="neg_mean_squared_log_error",n_iter=15 )
y_train = train[TARGET_COLUMN].values
y_aux_train = train[AUX_COLUMNS].values<define_variables>
%%time
grid_pipe_xgboost.fit(X_train,y_train )
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4 
<train_model>
grid_pipe_xgboost.best_params_
start_time = time.time()
print("Start training...")
 checkpoint_predictions = []
weights = []
for model_idx in range(1):
model = build_model_simplelstm(embedding_matrix_glove, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 train_word_sequences,
[y_train, y_aux_train],
batch_size=400,
epochs=1,
verbose=1,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(test_word_sequences, batch_size=2048)[0].flatten())
 result_ = pd.DataFrame.from_dict({
'id': test.id,
'prediction': checkpoint_predictions[-1]
})
 result_.to_csv("model_%d_%d.csv" %(model_idx, global_epoch))
weights.append(2 ** global_epoch)
 print("--- %s seconds ---" %(time.time() - start_time))<save_to_csv>
grid_pipe_xgboost.best_score_
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 submission = pd.DataFrame.from_dict({
'id': test.id,
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False )<train_model>
pipe_predict_xgboost = grid_pipe_xgboost.predict(X_test )
<import_modules>
from sklearn.metrics import mean_squared_log_error
import gc
import numpy as np
import pandas as pd
from typing import List, Tuple, Dict
from keras.models import Model
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler<define_variables>
print(np.sqrt(mean_squared_log_error(y_test,pipe_predict_xgboost)) )
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
NUM_MODELS = 2
EPOCHS = 4
BATCH_SIZE = 256
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'<categorify>
import lightgbm as lgb
def get_coefs(word: str, *arr: str)->(str, np.ndarray):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path: str)-> Dict[str, np.ndarray]:

 with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index: Dict[str, int], path: str, indexesOfWordsContainTrump: List[int])-> np.ndarray:

 embedding_dict = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
if(i in indexesOfWordsContainTrump):
embedding_matrix[i] = embedding_dict['Trump']
else:
try:
embedding_matrix[i] = embedding_dict[word]
except KeyError:
pass
return embedding_matrix
def build_model(embedding_matrix: np.ndarray)-> Model:

 words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(len(AUX_COLUMNS), activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<prepare_x_and_y>
pipeline_lgm = Pipeline([ 
('featureunion',combined_pipeline), 
('dictifier',FunctionTransformer(func=return_dict,validate=False)) , 
('vectorizer',DictVectorizer(sort=False,sparse=False)) , 
('reg_lgb',lgb.LGBMRegressor())
 ] )
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str)
 for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.5, True, False)
 tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
%%time
pipeline_lgm.fit(X_train,y_train )
sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN] &(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])& train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN] & train_df['homosexual_gay_or_lesbian'] + 0)* 5
sample_weights +=(~train_df[TARGET_COLUMN] & train_df['black'] + 0)* 5
sample_weights +=(~train_df[TARGET_COLUMN] & train_df['white'] + 0)* 5
sample_weights +=(~train_df[TARGET_COLUMN] & train_df['muslim'] + 0)* 1
sample_weights +=(~train_df[TARGET_COLUMN] & train_df['jewish'] + 0)* 1
sample_weights /= sample_weights.mean()<define_variables>
pipeline_lgm_predict = pipeline_lgm.predict(X_test)
 print(np.sqrt(mean_squared_log_error(y_test,pipeline_lgm_predict)) )
indexesOfWordsContainTrump = []
for word, index in tokenizer.word_index.items() :
if(( 'trump' in word)or('Trump' in word)) :
indexesOfWordsContainTrump.append(index)
 embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, filePath, indexesOfWordsContainTrump)for filePath in EMBEDDING_FILES], axis=-1 )<drop_column>
params_lgb = {
'reg_lgb__learning_rate' : [.1,.2,.3,.4,.5,.6],
'reg_lgb__n_estimators' : [50,100,200],
'reg_lgb__colsample_bytree' : [.1,.2,.3,.4,.5,.6,.7],
'reg_lgb__reg_lambda' : np.arange(0,0.6,0.1)
 }
del train_df
del tokenizer
gc.collect()<train_model>
grid_pipe_lgm = RandomizedSearchCV(estimator=pipeline_lgm,param_distributions=params_lgb,cv=4,
scoring="neg_mean_squared_log_error",n_iter=15 )
checkpoint_predictions = []
weights = []
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.6 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 predictions = np.average(checkpoint_predictions, weights=weights, axis=0 )<save_to_csv>
%%time
grid_pipe_lgm.fit(X_train,y_train )
submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False )<define_variables>
grid_pipe_lgm.best_params_
package_dir = ".. /input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT"
sys.path.append(package_dir )<set_options>
grid_pipe_lgm.best_score_
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<string_transform>
lgm_pipe_predict = grid_pipe_lgm.predict(X_test)
 print(np.sqrt(mean_squared_log_error(y_test,lgm_pipe_predict)) )
def convert_lines(example, max_seq_length,tokenizer):
max_seq_length -=2
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"]+tokens_a+["[SEP]"])+[0] *(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens)
 def is_interactive() :
return 'SHLVL' not in os.environ
def seed_everything(seed=123):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
emb_arr = KeyedVectors.load(path)
 return emb_arr
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( max_features + 1, 300))
unknown_words = []
for word, i in word_index.items() :
if i <= max_features:
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.lower() ]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.title() ]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
def sigmoid(x):
return 1 /(1 + np.exp(-x))
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds
def handle_punctuation(x):
x = x.translate(remove_dict)
 x = x.translate(isolate_dict)
 return x
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def preprocess(x):
x = handle_punctuation(x)
 x = handle_contractions(x)
 x = fix_quote(x)
 return x
class SequenceBucketCollator() :
def __init__(self, choose_length, sequence_index, length_index, label_index=None):
self.choose_length = choose_length
self.sequence_index = sequence_index
self.length_index = length_index
self.label_index = label_index
def __call__(self, batch):
batch = [torch.stack(x)for x in list(zip(*batch)) ]
sequences = batch[self.sequence_index]
lengths = batch[self.length_index]
length = self.choose_length(lengths)
 mask = torch.arange(start=maxlen, end=0, step=-1)< length
padded_sequences = sequences[:, mask]
batch[self.sequence_index] = padded_sequences
if self.label_index is not None:
return [x for i, x in enumerate(batch)if i != self.label_index], batch[self.label_index]
return batch
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x, lengths=None):
h_embedding = self.embedding(x.long())
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
def custom_loss(data, targets):
bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2
def reduce_mem_usage(df):
start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
def ensemble_predictions(predictions, weights, type_="linear"):
assert np.isclose(np.sum(weights), 1.0)
 if type_ == "linear":
res = np.average(predictions, weights=weights, axis=0)
 elif type_ == "harmonic":
res = np.average([1 / p for p in predictions], weights=weights, axis=0)
 return 1 / res
elif type_ == "geometric":
numerator = np.average(
 [np.log(p)for p in predictions], weights=weights, axis=0 
)
 res = np.exp(numerator / sum(weights))
return res
elif type_ == "rank":
res = np.average([rankdata(p)for p in predictions], weights=weights, axis=0)
 return res /(len(res)+ 1)
 return res<set_options>
pipeline_elasticnet = Pipeline([ 
('featureunion',combined_pipeline), 
('dictifier',FunctionTransformer(func=return_dict,validate=False)) , 
('vectorizer',DictVectorizer(sort=False,sparse=False)) , 
('reg_en',ElasticNet())
 ] )
warnings.filterwarnings(action='once')
 device = torch.device('cuda')
 MAX_SEQUENCE_LENGTH = 300
SEED = 1234
BATCH_SIZE = 512
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
bert_config = BertConfig('.. /input/bert-inference/bert/bert_config.json')
 tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True)
 tqdm.pandas()
CRAWL_EMBEDDING_PATH = '.. /input/gensim-embeddings-dataset/crawl-300d-2M.gensim'
GLOVE_EMBEDDING_PATH = '.. /input/gensim-embeddings-dataset/glove.840B.300d.gensim'
NUM_MODELS = 8
LSTM_UNITS = 512
DENSE_HIDDEN_UNITS = 16 * LSTM_UNITS
MAX_LEN = 220
if not is_interactive() :
def nop(it, *a, **k):
return it
tqdm = nop
fastprogress.fastprogress.NO_BAR = True
master_bar, progress_bar = force_console_behavior()
fastai.basic_train.master_bar, fastai.basic_train.progress_bar = master_bar, progress_bar
seed_everything()<load_from_csv>
pipeline_elasticnet.fit(X_train,y_train )
test_df = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv")
 test_df['comment_text'] = test_df['comment_text'].astype(str)
 X_test = convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), MAX_SEQUENCE_LENGTH, tokenizer )<load_pretrained>
predicted_base_elastic_net = pipeline_elasticnet.predict(X_test)
 print(np.sqrt(mean_squared_log_error(y_test,predicted_base_elastic_net)) )
model = BertForSequenceClassification(bert_config, num_labels=1)
 model.load_state_dict(torch.load(".. /input/bert-inference/bert/bert_pytorch.bin"))
model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()<categorify>
params_elasticnet = {
'reg_en__l1_ratio' : [0.2,0.4,0.6,0.8],
'reg_en__alpha' : [0.5,1,1.5,2],
}
test_preds = np.zeros(( len(X_test)))
 test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=512, shuffle=False)
 tk0 = tqdm(test_loader)
 for i,(x_batch,)in enumerate(tk0):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 test_preds[i * 512:(i + 1)* 512] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()<save_to_csv>
grid_pipe_elastic = RandomizedSearchCV(estimator=pipeline_elasticnet,param_distributions=params_elasticnet,cv=4,
scoring="neg_mean_squared_log_error" )
submission_bert = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': test_pred
})
 submission_bert.to_csv('submission.csv', index=False)
<load_from_csv>
%%time
grid_pipe_elastic.fit(X_train,y_train )
def read_competision_file(train=True):
if train:
with open('.. /input/pickled-jigsaw-unintended-bias-in-toxicity-train/train.pickle','rb')as f:
df = pickle.load(f)
 else:
df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 return df<set_options>
grid_pipe_elastic.best_params_
def trans_text(s, dict):
return s.translate(dict)
 tknzr = TweetTokenizer(strip_handles=True)
 reg_twt = re.compile(r"(https:\/\/twitter[-_\.!~*'() a-zA-Z0-9;\/?:\@&=\+\$,%
reg_wik = re.compile(r"(https:\/\/[a-z][a-z]\.wikipedia\.[-_\.!~*'() a-zA-Z0-9;\/?:\@&=\+\$,%
reg_utb1 = re.compile(r"(https:\/\/youtu\.be[-_\.!~*'() a-zA-Z0-9;\/?:\@&=\+\$,%
reg_utb2 = re.compile(r"(https:\/\/www\.youtube\.com[-_\.!~*'() a-zA-Z0-9;\/?:\@&=\+\$,%
reg_url1 = re.compile(r"(https?|ftp )(:\/\/[-_\.!~*'() a-zA-Z0-9;\/?:\@&=\+\$,%
reg_url2 = re.compile(r"(bit\.ly\/[a-zA-Z0-9]+)")
 reg_url3 = re.compile(r"(goo\.gl\/[a-zA-Z0-9]+)")
 reg_url4 = re.compile(r"(www\.[a-zA-Z][a-zA-Z0-9\.]*[a-zA-Z0-9]+\.( com|org|net|int|edu|gov|mil|([a-z][a-z])))")
 reg_tid = re.compile(r"(@[a-zA-Z0-9]+)")
 reg_tag = re.compile(r"(
 reg_num = re.compile(r"([0-9][0-9]+)")
 prefix_word_list = ['trans','super','hyper','every','cyber','anti','bull','and','any','pre','non','sub','car','dis','mis','neo','at','un','ex','be','co','re','de','by']
emoji_isolate_dict = {'(8':'🙂', '(:':'🙂', '(:>':'🙂', '(;':'🙂', '(=':'🙂', '(=>':'🙂', ')';':'🙁', ')8':'🙂', '):':'🙁', ');':'🙁', ')=':'🙁', ':'(':'🙂', ':')':'🙂', ':'D':'🙂', ':'d':'😜', ':(':'🙁', ':)':'🙂', ':*(':'🙁', ':*)':'🙂', ':8':'🙂', ':;':'🙁', ':=':'🙂', ':D':'🙂',':O(':'🙂', ':O)':'🙂', ':Op':'😜', ':P':'😜', ':[':'🙁', ':'':'🙁', ':]':'🙂', ':d':'😜', ':o':'🙂', ':o(':'🙁', ':o)':'🙂', ':oD':'🙂', ':o'':'🙁', ':op':'😜', ':p':'😜', ':|':'🙁', ':}':'🙂', ';')':'🙂', ';(':'🙁', ';)':'🙂', ';:':'🙁', ';D':'🙂', ';O)':'🙂', ';P':'😜', ';[':'🙁', ';]':'🙂', ';d':'😜', ';o(':'🙁', ';o)':'🙂', ';oD':'🙂', ';oP':'😜', ';od':'😜', ';op':'😜', ';o}':'🙂', ';p':'😜', ';{':'🙁', ';}':'🙂', '=(':'🙁', '=)':'🙂', '=:':'🙁', '=D':'🙂', '=Op':'😜', '=P':'😜', '='':'🙁', '=d':'😜', '=o)':'🙂', '=op':'🙁', '=o}':'🙂', '=p':'😜', '=}':'🙂', '>:':'🙁', '>:(':'🙁', '>:)':'🙁', '>;':'🙂', '>=p':'😜', '@:':'🙂', '@;':'🙁', '{:':'🙂', '{;':'🙂', '{=':'🙂', '|:':'🙁', '}:':'🙁', '};':'🙁'}
separatedict = {ord(k):' ' for k in "!"'-
"}
def replace_text(s, deletedict, isolatedict):
s = re.sub(reg_twt, ' <twt> ' ,s)
 s = re.sub(reg_wik, ' <wik> ' ,s)
 s = re.sub(reg_utb1, ' <utb> ' ,s)
 s = re.sub(reg_utb2, ' <utb> ' ,s)
 s = re.sub(reg_url1, ' <url> ' ,s)
 s = re.sub(reg_url2, ' <url> ' ,s)
 s = re.sub(reg_url3, ' <url> ' ,s)
 s = re.sub(reg_url4, ' <url> ' ,s)
 s = re.sub(reg_tid, ' <tid> ' ,s)
 s = re.sub(reg_tag, ' <tag> ' ,s)
 s = s.replace('?', ' <qes> ')
 s = s.replace('
', ' <ret> ')
 s = re.sub(reg_num, ' <num> ' ,s)
 s = s.translate(deletedict)
 return s.translate(isolatedict)
 def getsimbols(s):
all_charactors = set()
for cc in str(s):
if ord(cc)>= 256:
all_charactors.add(cc)
 return all_charactors
stemmer = PorterStemmer()
@lru_cache(120000)
 def stem(s):
return stemmer.stem(s)
 @lru_cache(120000)
 def corr(s):
return str(Word(s ).correct())
 def get_til(s, dict_words):
if s in dict_words:
return s
s_t = s.title()
if s_t in dict_words:
return s_t
s_l = s.lower()
if s_l in dict_words:
return s_l
return None
def get_stil(s, dict_words):
t = get_til(s, dict_words)
 if t is not None:
return t
s = stem(s)
 t = get_til(s, dict_words)
 if t is not None:
return t
def get_heads(s, pref, dict_words):
p = pref + '-' + s[len(pref):]
t = get_stil(p, dict_words)
 if t is not None:
return [t]
p = pref + '-' + s[len(pref):].lower()
t = get_stil(p, dict_words)
 if t is not None:
return [t]
p = pref + '-' + s[len(pref):].title()
t = get_stil(p, dict_words)
 if t is not None:
return [t]
p = pref + '-' + stem(s[len(pref):])
 t = get_til(p, dict_words)
 if t is not None:
return [t]
p = pref + '-' + stem(s[len(pref):].lower())
 t = get_til(p, dict_words)
 if t is not None:
return [t]
p = pref + '-' + stem(s[len(pref):].title())
 t = get_til(p, dict_words)
 if t is not None:
return [t]
t1 = get_til(pref, dict_words)
 t2 = get_stil(s[len(pref):], dict_words)
 if t1 is not None and t2 is not None:
return [t1,t2]
return None
def get_stem(s, dict_words, spell_collector):
t = get_stil(s, dict_words)
 if t is not None:
return [t]
if '-' in s:
t = get_stil(s.replace('-',''), dict_words)
 if t is not None:
return [t]
if ,''), dict_words)
 if t is not None:
return [t]
for pref in prefix_word_list:
if len(s)> len(pref)+1 and s.lower().startswith(pref):
tl = get_heads(s, pref, dict_words)
 if tl is not None:
return tl
if type(spell_collector)== dict:
if s in spell_collector:
p = spell_collector[s]
t = get_stil(p, dict_words)
 if t is not None:
return [t]
elif str(type(spell_collector)) == "<class 'function'>":
p = spell_collector(s)
 t = get_stil(p, dict_words)
 if t is not None:
return [t]
elif type(spell_collector)== str and spell_collector == 'textblob':
p = corr(s)
 t = get_stil(p, dict_words)
 if t is not None:
return [t]
return None
def fix_spell(s, dict_words, spell_collector):
if s in emoji_isolate_dict:
return [emoji_isolate_dict[s]]
tl = get_stem(s, dict_words, spell_collector)
 if tl is not None:
return tl
t = s.translate(separatedict)
 if t != s:
v = [get_stem(u, dict_words, spell_collector)for u in t.split() ]
return sum([w for w in v if w is not None],[])
 return [s]
def tokenize_text(s, dict_words, spell_collector):
return sum([fix_spell(w, dict_words, spell_collector)for w in tknzr.tokenize(s)],[])
 class TokenVectorizer:
def __init__(self, vector_dict, uniform_unknown_word=False, lemma_dict={}, spell_collector='textblob', num_process=-1):
self.vector_dict = vector_dict
self.uniform_unknown_word = uniform_unknown_word
self.lemma_dict = lemma_dict
self.spell_collector = spell_collector
self.num_process = num_process if num_process>0 else os.cpu_count()
self.word_index = dict()
self.unknown_words = set()
if type(self.vector_dict)== str:
self.load_vector_dict(self.vector_dict)
 if type(self.lemma_dict)== str:
if os.path.isfile(self.lemma_dict):
with open(self.lemma_dict)as f:
self.lemma_dict = json.load(f)
 else:
self.lemma_dict = dict()
if type(self.spell_collector)== str:
if os.path.isfile(self.spell_collector):
with open(self.spell_collector)as f:
self.spell_collector = json.load(f)
 self.deletedict = dict()
self.isolatedict = dict()
def load_vector_dict(self, vector_file):
if vector_file.endswith('.pickle')or vector_file.endswith('.pkl'):
with open(vector_file, 'rb')as f:
self.vector_dict = pickle.load(f)
 elif vector_file.endswith('.bz2'):
with bz2.BZ2File(vector_file, 'rb')as f:
self.vector_dict = pickle.loads(f.read())
 else:
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 with open(vector_file)as f:
self.vector_dict = dict(get_coefs(*line.strip().split(' ')) for line in f)
 if '<url>' not in self.vector_dict:
v = self.vector_dict.values()
mat_size = np.max([len(t)for t in v])
 emb_mean, emb_std = np.mean([np.mean(t)for t in v]), np.mean([np.std(t)for t in v])
 np.random.seed(12)
 self.vector_dict['<tid>'] = np.random.normal(emb_mean, emb_std,(mat_size,))
self.vector_dict['<tag>'] = np.random.normal(emb_mean, emb_std,(mat_size,))
self.vector_dict['<qes>'] = np.random.normal(emb_mean, emb_std,(mat_size,))
self.vector_dict['<ret>'] = np.random.normal(emb_mean, emb_std,(mat_size,))
self.vector_dict['<twt>'] = np.random.normal(emb_mean, emb_std,(mat_size,))
self.vector_dict['<wik>'] = np.random.normal(emb_mean, emb_std,(mat_size,))
self.vector_dict['<utb>'] = np.random.normal(emb_mean, emb_std,(mat_size,))
self.vector_dict['<url>'] = np.random.normal(emb_mean, emb_std,(mat_size,))
self.vector_dict['<num>'] = np.random.normal(emb_mean, emb_std,(mat_size,))
def preprocess(self, string_list):
lenmadict = {ord(k):v for k,v in self.lemma_dict.items() }
all_simbols = set()
dict_words = set(self.vector_dict.keys())
 if self.num_process <= 1:
if len(lenmadict)> 0:
string_list = [s.translate(lenmadict)for s in string_list]
for s in string_list:
all_simbols |= getsimbols(s)
 else:
if len(lenmadict)> 0:
with Pool(self.num_process)as pl:
r = pl.starmap(trans_text, [(s,lenmadict)for s in string_list ], chunksize=10000)
 for i,p in enumerate(r):
string_list[i] = p
with Pool(self.num_process)as pl:
r = pl.imap_unordered(getsimbols, string_list, chunksize=10000)
 for p in r:
all_simbols |= p
self.deletedict = {ord(k):'' for k in(all_simbols - dict_words)}
self.isolatedict = {ord(k):' %s '%k for k in(dict_words & all_simbols)}
if self.num_process <= 1:
for i,s in enumerate(string_list):
string_list[i] = replace_text(s,self.deletedict,self.isolatedict)
 else:
with Pool(self.num_process)as pl:
r = pl.starmap(replace_text, [(s,self.deletedict,self.isolatedict)for s in string_list ], chunksize=10000)
 for i,p in enumerate(r):
string_list[i] = p
return string_list
def tokenize(self, string_list, maxlen=-1, pad_sequence=True):
dict_words = set(self.vector_dict.keys())
 all_seq = []
for s in string_list:
all_seq.append([])
 if self.num_process <= 1:
for i,s in enumerate(string_list):
all_seq[i] = tokenize_text(s, dict_words, self.spell_collector)
 else:
with Pool(self.num_process)as pl:
r = pl.starmap(tokenize_text, [(s, dict_words, self.spell_collector)for s in string_list ], chunksize=10000)
 for i,t in enumerate(r):
all_seq[i] = t
for i,s_token in enumerate(all_seq):
tokens = []
for t in s_token:
if t in self.word_index:
tokens.append(self.word_index[t])
 else:
self.word_index[t] = len(self.word_index)+ 1
tokens.append(self.word_index[t])
 all_seq[i] = tokens
if pad_sequence:
if maxlen <= 0:
maxlen = np.max([len(t)for t in all_seq])
 for i,tokens in enumerate(all_seq):
if len(tokens)> maxlen:
all_seq[i] = tokens[len(tokens)-maxlen:]
elif len(tokens)< maxlen:
all_seq[i] = [0 for w in range(maxlen-len(tokens)) ] + tokens
return np.array(all_seq, dtype=np.int64)
 else:
if maxlen > 0:
for i,tokens in enumerate(all_seq):
if len(tokens)> maxlen:
all_seq[i] = tokens[len(tokens)-maxlen:]
return all_seq
def vectorize(self):
v = self.vector_dict.values()
mat_size = np.max([len(t)for t in v])
 if self.uniform_unknown_word:
emb_mean, emb_std = np.mean([np.mean(t)for t in v]), np.mean([np.std(t)for t in v])
 embedding_matrix = np.zeros(( len(self.word_index)+ 1, mat_size))
for word_o, i in self.word_index.items() :
if word_o in self.vector_dict:
embedding_matrix[i] = self.vector_dict[word_o]
else:
if self.uniform_unknown_word:
embedding_matrix[i] = np.random.normal(self.emb_mean, self.emb_std,(mat_size,))
self.unknown_words.add(word_o)
 return embedding_matrix
def __call__(self, string_list, maxlen=-1, pad_sequence=True):
print("Preprocess.")
 string_list = self.preprocess(string_list)
 print("Tokenize.")
 tokens = self.tokenize(string_list,maxlen,pad_sequence)
 print("Vectorize.")
 return tokens, self.vectorize()<load_pretrained>
grid_pipe_elastic.best_score_
def load_bert_vocab_as_embed(vocab_file='.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/vocab.txt'):

 vocab = dict()
index = 0
with open(vocab_file, "r", encoding="utf-8")as reader:
while True:
token = reader.readline()
if not token:
break
token = token.strip()
vocab[token] = [index]
index += 1
return vocab<string_transform>
from sklearn.linear_model import Lasso
def convert_lines(example, max_seq_length, tokenizer):
max_seq_length -=2
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"]+tokens_a+["[SEP]"])+[0] *(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens )<load_pretrained>
from sklearn.linear_model import Lasso
def get_bert_tokens() :
tokenizer = BertTokenizer.from_pretrained('.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/', cache_dir=None,do_lower_case=True)
 test_df = read_competision_file(False)
 test_df['comment_text'] = test_df['comment_text'].astype(str)
 return convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), 380, tokenizer)
<load_pretrained>
lasso = Lasso()
def get_bert_model() :
device = torch.device('cuda')
 bert_config = BertConfig('.. /input/bert-inference/bert/bert_config.json')
 model = BertForSequenceClassification(bert_config, num_labels=1)
 model.load_state_dict(torch.load(".. /input/bert-inference/bert/bert_pytorch.bin"))
model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
return model<categorify>
pipeline_lasso = Pipeline([ 
('featureunion',combined_pipeline), 
('dictifier',FunctionTransformer(func=return_dict,validate=False)) , 
('vectorizer',DictVectorizer(sort=False,sparse=False)) , 
('reg_lasso',Lasso())
 ] )
def make_bert_submission(output_file):
device = torch.device('cuda')
 all_sent = get_bert_tokens()
model = get_bert_model()
test_preds = np.zeros(( len(all_sent)))
 print(test_preds.shape)
 test = torch.utils.data.TensorDataset(torch.tensor(all_sent, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=128, shuffle=False)
 for i,(x_batch,)in enumerate(test_loader):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 sp = i * 128
ep = sp + len(pred)
 test_preds[sp:ep] = pred[:, 0].detach().cpu().squeeze().numpy()
predictions = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()
df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df['prediction'] = predictions
df.to_csv(output_file, index=False )<import_modules>
params_lasso = {
'reg_lasso__alpha' : np.logspace(-4, -3, 5),
}
required = object()
class Optimizer(object):

 def __init__(self, params, defaults):
self.defaults = defaults
if isinstance(params, Variable)or torch.is_tensor(params):
raise TypeError("params argument given to the optimizer should be "
"an iterable of Variables or dicts, but got " +
torch.typename(params))
self.state = defaultdict(dict)
 self.param_groups = []
param_groups = list(params)
 if len(param_groups)== 0:
raise ValueError("optimizer got an empty parameter list")
 if not isinstance(param_groups[0], dict):
param_groups = [{'params': param_groups}]
for param_group in param_groups:
self.add_param_group(param_group)
 def __getstate__(self):
return {
'state': self.state,
'param_groups': self.param_groups,
}
def __setstate__(self, state):
self.__dict__.update(state)
 def state_dict(self):

 def pack_group(group):
packed = {k: v for k, v in group.items() if k != 'params'}
packed['params'] = [id(p)for p in group['params']]
return packed
param_groups = [pack_group(g)for g in self.param_groups]
packed_state = {(id(k)if isinstance(k, Variable)else k): v
for k, v in self.state.items() }
return {
'state': packed_state,
'param_groups': param_groups,
}
def load_state_dict(self, state_dict):

 state_dict = deepcopy(state_dict)
 groups = self.param_groups
saved_groups = state_dict['param_groups']
if len(groups)!= len(saved_groups):
raise ValueError("loaded state dict has a different number of "
"parameter groups")
 param_lens =(len(g['params'])for g in groups)
 saved_lens =(len(g['params'])for g in saved_groups)
 if any(p_len != s_len for p_len, s_len in zip(param_lens, saved_lens)) :
raise ValueError("loaded state dict contains a parameter group "
"that doesn't match the size of optimizer's group")
 id_map = {old_id: p for old_id, p in
zip(chain(*(g['params'] for g in saved_groups)) ,
chain(*(g['params'] for g in groups)))}
def cast(param, value):

 if torch.is_tensor(value):
if any(tp in type(param.data ).__name__ for tp in {'Half', 'Float', 'Double'}):
value = value.type_as(param.data)
 value = value.cuda(param.get_device())if param.is_cuda else value.cpu()
return value
elif isinstance(value, dict):
return {k: cast(param, v)for k, v in value.items() }
elif isinstance(value, Iterable):
return type(value )(cast(param, v)for v in value)
 else:
return value
state = defaultdict(dict)
 for k, v in state_dict['state'].items() :
if k in id_map:
param = id_map[k]
state[param] = cast(param, v)
 else:
state[k] = v
def update_group(group, new_group):
new_group['params'] = group['params']
return new_group
param_groups = [
update_group(g, ng)for g, ng in zip(groups, saved_groups)]
self.__setstate__({'state': state, 'param_groups': param_groups})
 def zero_grad(self):

 for group in self.param_groups:
for p in group['params']:
if p.grad is not None:
data = p.grad.data
p.grad = Variable(data.new().resize_as_(data ).zero_())
 def step(self, closure):

 raise NotImplementedError
def add_param_group(self, param_group):

 assert isinstance(param_group, dict), "param group must be a dict"
params = param_group['params']
if isinstance(params, Variable):
param_group['params'] = [params]
else:
param_group['params'] = list(params)
 for param in param_group['params']:
if not isinstance(param, Variable):
raise TypeError("optimizer can only optimize Variables, "
"but one of the params is " + torch.typename(param))
if not param.requires_grad:
raise ValueError("optimizing a parameter that doesn't require gradients")
 if not param.is_leaf:
raise ValueError("can't optimize a non-leaf Variable")
 for name, default in self.defaults.items() :
if default is required and name not in param_group:
raise ValueError("parameter group didn't specify a value of required optimization parameter " +
name)
 else:
param_group.setdefault(name, default)
 param_set = set()
for group in self.param_groups:
param_set.update(set(group['params']))
if not param_set.isdisjoint(set(param_group['params'])) :
raise ValueError("some parameters appear in more than one parameter group")
 self.param_groups.append(param_group)
 class Nadam(Optimizer):

 def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps,
weight_decay=weight_decay)
 super(Nadam, self ).__init__(params, defaults)
 def step(self, lr=None, closure=None):

 loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data
if grad.is_sparse:
raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
 state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p.data)
 state['exp_avg_sq'] = torch.zeros_like(p.data)
 state['prod_mu_t'] = 1.
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
state['step'] += 1
if group['weight_decay'] != 0:
grad = grad.add(group['weight_decay'], p.data)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 mu_t = beta1*(1 - 0.5*0.96**(state['step']/250))
mu_t_1 = beta1*(1 - 0.5*0.96**(( state['step']+1)/250))
prod_mu_t = state['prod_mu_t'] * mu_t
prod_mu_t_1 = prod_mu_t * mu_t_1
state['prod_mu_t'] = prod_mu_t
g_hat = grad/(1-prod_mu_t)
 m_hat = exp_avg /(1-prod_mu_t_1)
 m_bar =(1-mu_t)*g_hat + mu_t_1*m_hat
exp_avg_sq_hat = exp_avg_sq/(1 - beta2 ** state['step'])
 denom = exp_avg_sq_hat.sqrt().add_(group['eps'])
 bias_correction1 = 1 - beta1 ** state['step']
bias_correction2 = 1 - beta2 ** state['step']
if lr is not None:
step_size = lr
else:
step_size = group['lr']
p.data.addcdiv_(-step_size, m_bar, denom)
 return loss 
<define_variables>
grid_pipe_lasso = RandomizedSearchCV(estimator=pipeline_lasso,param_distributions=params_lasso,cv=4,
scoring="neg_mean_squared_error" )
def make_isolated_embedding(matrix):
p_matrix = [0.009182812785100022,-0.040584789317261394,-0.011846708215597239,-0.03270542198528366,0.005831483538662122,-0.0004415112849428413,-0.05269573502126941,0.008208912498716938,0.019490382223248188,-0.02458577452404483,0.028250662928277572,0.18361226878326614,0.021729090546093142,-0.038275015170959006,0.0033973759346653093,0.008467473701057892,-0.028189553307668806,0.024824375401442614,0.0036222467359421444,0.024460037156900954,-0.02378269272174318,-0.01817182542976324,-0.019882241470596283,-0.031097727239060385,-0.00599710407291916,-0.03152349409585279,0.009122488615571177,-0.0001713773609040107,-0.01644240735678459,0.03454689363198803,0.005078502134341189,-0.004377185795367971,0.0048557909975371835,-0.08006686829112608,-0.003760243833561533,-0.0248445653734306,-0.017009404930426887,-0.00569534237879755,-0.000642721609025357,0.03659293694138439,-0.009366632345025928,0.043001355613987345,0.014811545442659625,-0.0553467682511432,0.032799926786718746,-0.03219448863992589,0.03322567142992652,-0.026552534013046455,0.007767168305946927,0.02143737139128056,-0.012692011632648841,-0.06745580507468553,0.2515678896580645,-0.0013208436697328095,-3.301024508568336e-05,-0.0003015832384097134,-0.019003431418042088,-0.00188837459088892,0.00436508684388134,0.011050059997033708,-0.001197451984162603,0.009456930867826065,-0.007320853927069258,-0.09904656272567001,-0.0007289314432978023,0.010478639940070549,-0.00046170855686492897,-0.0018897862799705932,-0.014744774384683417,-0.004896648364511924,-0.04493761518876994,-0.010619764907179256,-0.008544664017908073,0.1375873417196435,-0.011918515484566967,-0.019944713637476748,-0.44255750737304644,-0.02587336684423643,0.09539878700496177,-0.0008147322754549869,0.43128423586384984,-0.0024291757497341086,0.007918524104226929,0.005201713692837981,-0.028053819611239265,-0.012091343156606765,-0.000799473765963778,0.009598349126455482,-0.02050136027628646,0.0032380949567738524,0.0018364292346734206,0.22967682721094831,0.011433104003041313,0.013457276251713958,-0.019701611001235957,-0.0321900493476438,0.01349966601716265,-0.018782597335088132,-0.017765461514827142,-0.023883732314340853,-0.008347606023424563,-0.012389906501522161,0.047805364109126385,-0.008904211179667242,0.005456968065525838,-0.013100424607604846,-0.007764529619285597,-0.008379769811379586,-0.0006800349414373718,0.035364785075270515,0.04445204126762924,-0.026929967458287473,-0.00202610635592915,0.03365310627309023,-0.43408399394956093,-0.022089394180255996,0.013561671001893557,0.010285446738670448,0.01582157729764278,-0.018236547325153906,0.00862505152209495,0.014860190596372995,0.02103445593325409,-0.013663182530340216,0.006236041163585062,-0.031506081879061607,0.02029873972278035,0.007525412512284544,0.2168811591908896,-0.029069495233385707,0.002299838727800742,-0.0061253794425330755,0.016092624984054584,0.004648741671515148,0.03265056138237865,-0.020335595885716887,-0.04517213831750648,-0.016729069807001527,0.02046132553829288,-0.22529141041734044,0.00387213584272845,-0.04590042121987848,0.010424768741486743,0.15805529411390468,-0.003857944676256511,-0.012855499964437788,-0.0031479227685063827,0.01376401982408701,-0.0008840460016189583,0.009722078630268836,-0.0039317025854035195,-0.019016283723644656,0.018775732975998105,-0.017937779275020417,0.031579564569935,-0.007669803337988481,-0.01687133659116447,-0.0221729483049514,0.029318401603712657,-0.005841326041278884,0.4990321811786259,0.009309199701765958,-0.02190627186795492,-0.0013635854455670434,-0.002199810216732186,-0.001540567758101668,0.03244679634465029,-0.02243383997032224,0.029107571828807473,-0.0034612887306351334,-0.01922863481025543,0.033475337590504854,0.027698550868682285,-0.018233073046847306,0.029581502701051407,-0.011272991897479179,-0.00635325244461321,-0.005290278157336871,-0.0012130241688268829,-0.01570584513601337,0.05570008294731479,0.02094458604337456,0.0027758341978849194,0.012296298175652344,0.007278864172280131,0.002402990513916204,0.014286256434092448,-0.027527592602248567,-0.027846720662737783,0.020494460118314867,0.009517022621249824,0.006147198375049207,-0.029689002775387656,0.012213014702094396,-0.005832880658189614,0.002928135512058671,0.01593060614654866,-0.009247318717706099,-0.048770400843015524,0.0070362227117428695,-0.03104945691093978,-0.015234546146962798,0.01427756671938701,0.007851132197927001,-0.010332352055422505,0.01787464251743762,-0.005924500741402286,-0.009247275418077844,0.050408742176736604,-0.025433062568983734,0.033073633640458254,0.5594965267559358,0.02002025955553414,0.006537861022725837,0.020348736349184635,-0.05122281582737965,0.008732368733769882,0.007584178345455834,-0.008786773077186328,0.024175882003831195,-0.02915484864102269,0.17608653925668571,-0.0022359680364203247,-0.016674727443089767,0.02473502033309728,0.024462908760058428,0.0021528612143051967,0.019582923925334672,-0.011121453273058109,0.022039536653415445,-0.018339546014538938,-0.008183806335540509,0.003427008496014203,0.022422575067738395,-0.010886776970292125,-0.14427446303533234,-0.01418545773270993,-0.03238430572051303,0.002060891925426355,-0.002544074362113623,-0.012750695242480304,-0.026500782326316006,-0.019240183126669766,0.01914396654898081,-0.06365170442740913,0.0010214308427048664,0.05724444478284752,0.01443026011061149,0.010996966732755688,0.014228829194472964,0.004425919209124025,0.023091573204805673,-0.01407635056188351,0.006131513263332672,0.017010445713140893,0.028046560357275086,0.011752955298156946,-0.03539766241703242,0.018111176525338966,-0.009629944099677398,0.005488088929932913,0.024416553691102957,0.00799905391368688,-0.03976269392045281,-0.01445453848740384,0.008963378241604711,0.0027252719975425737,0.018520746431111717,-0.051008665560630814,-0.0276821147291433,-0.0031133075706395036,-0.013730677829329312,-0.00988280967096081,-0.022789165765726507,-0.013418135885518098,-0.016176950139522125,0.029539708971903212,0.0023987474714475907,0.0007228514612702998,0.001883056702848162,2.8693485735020397e-05,0.022819038719717185,-0.011042038850970244,0.0016225553727346816,0.000765848399744972,-0.015733646468002856,0.0002460813122046364,0.004383365873786323,0.0018234625955432856,0.008816511465642465,-0.004697932903667204,0.017899667797015045,0.016273725769387452,-0.007007963750189115,-0.006003545090912512,-0.051004466413320376,-0.006022282213469877,-0.015093700763854715,-0.019613736605590657,0.02501420948335617]
n_matrix = [0.002427948938680964,-0.026350857341430108,-0.01523373394668481,-0.028005489746274863,0.004488919278566633,0.0025764452876863887,-0.05505243786917943,0.011080624352782405,0.018293541273728998,-0.01916369048950003,0.011849926040022529,0.1726957534637598,0.02671063208648816,-0.0369202165242552,0.002791046706245471,0.002561140139550723,-0.014765278434723918,0.02653718321829409,0.007576875917334894,0.02357230384341765,-0.0209466817996944,-0.022988040990450615,-0.01977003211455475,-0.020617486769208188,-0.008075158050003514,-0.0179954635366672,0.006806415834599482,0.0011730936942883922,-0.012051379742019114,0.010921753093303508,-0.001154570365234572,-0.008862633087375532,0.0027804213809410126,-0.07781073560502591,-0.004484659507214222,-0.023580608532469592,-0.020276033932850976,0.0014621975587638385,-0.0048490894033213884,0.035544478277694906,-0.010430916969976944,0.040756492271460484,0.012992691111281212,-0.06436971529312906,0.025517813396807757,-0.02138459024632788,0.020928812631391694,-0.022389823873997926,0.008460295061238595,0.04025238390058873,-0.014675050104351021,-0.05888850239540475,0.24121504093944438,-0.009924799986949144,-0.002962292819389109,0.010771495266329164,-0.014973904330258992,-0.010481301415531025,0.00904944982868231,0.014351674629124914,-0.0017139671479696242,0.0022342122900886638,-0.008853865242500552,-0.10524972617760524,-0.012833226056431994,0.006743596493284535,0.0007670770004786059,0.006012935550680465,-0.010122846041464167,0.006084273449751132,-0.04452407145501877,0.0007107327684431172,-0.006312375965486463,0.1182880331765001,-0.016177655874426787,-0.018447948985036645,-0.4424178339546986,-0.018872906542617345,0.08897691119673731,-0.00470415901595502,0.436660468941245,0.0032877133727051045,0.009196369954346289,-0.002068010661233682,-0.025031863949072013,-0.00926069690416934,0.005462458250593397,0.0021068353560220665,-0.01709488146118999,-0.0029566709210736507,0.000943620246203871,0.2317172252976802,0.008806959870494525,0.00752088202903455,-0.01449295517566327,-0.017681022252292778,0.016020909247113465,-0.01145472112074367,-0.010618605714498804,-0.02522631336859578,-0.004938079177010351,-0.008660230542255298,0.054626019504476805,-0.002525091526663276,-0.005733962870373745,-0.005273452676593823,-0.0065782046329970565,-0.00017058415938834653,-0.0027915610957764763,0.03064282316411414,0.03284994307001055,-0.020904405426359192,-0.00020843298911267657,0.029267702853651444,-0.4260006183631928,-0.02237897081887229,0.010642088237609242,0.010763545739179573,0.0262600524725892,-0.011103068722273873,-0.0016078490097023298,0.010304944165445345,0.019162263645207865,-0.02067159837124151,0.0031207210034887908,-0.02843001930552024,0.012618934736768568,0.011433547789040035,0.22596976546798553,-0.03266044659799642,0.0018090003443324589,-0.013437553406251112,0.012028592313833273,0.001580889141141905,0.03961032739943911,-0.021245330451863238,-0.057640725834056314,-0.015443654799910172,0.01206629875735433,-0.21788295041073666,0.004999618692175123,-0.03360491136628174,0.012228892379779128,0.15182361061559915,0.0024251984860948427,-0.01863088311584819,-0.006704304497265216,0.003544175694853773,-0.010097172415480617,0.011831440400283454,0.0015263952300872186,-0.014214391913847595,0.0129691540697708,-0.016836943518607983,0.035253697888307636,-0.013735298400909429,-0.01119266674685763,-0.006467457637194071,0.021054150995139506,-0.00837039441004984,0.4996003039284626,0.01653211540808042,-0.021174870081088575,-0.0032856912452447007,0.0036857646677837894,0.007159742275286415,0.02419119789276778,-0.018161404557629695,0.025356454604348568,0.005035569901911509,-0.015101742715900165,0.016950226810508826,0.016419820357092434,-0.015038229616722236,0.020945601432537524,-0.009955255839281368,-0.0063248732964104265,-0.006259977931110731,-0.009253129006096147,-0.012844042192023921,0.06025393859641254,0.021554418742506565,-0.0022603530022002306,0.008646393059816073,0.0021184950436328813,-0.004328794379672818,0.0042814588130751455,-0.02360843975716069,-0.02270469406733238,0.027494024616281223,0.007354786948958046,0.0011223183696831356,-0.0246659101181,0.001192645231373174,-0.010403856681650388,-0.0008443164548717201,0.022717614738114334,-0.01816314633264307,-0.033544480214339985,0.011123094908872894,-0.021903283900440586,-0.01100938891326716,0.014690020458654723,0.005771770872135895,-0.009854506337303412,0.011973244880088087,-0.010742825727213726,-0.0026397026205206725,0.04712089776265658,-0.018975396543114788,0.03333279000104644,0.558984739978566,0.020200819094475533,0.006675996046620084,0.018603716363358665,-0.030034901499923806,0.005679375227906664,0.00404851041665811,-0.004779768929533408,0.019002550900477273,-0.021261162361613437,0.17052447262529102,0.000436158718672102,-0.00812029326320723,0.014730889369140389,0.01724511392048127,-0.0039528863298191,0.02100576562004079,-0.012447644038238661,0.010956277262089958,-0.02094148069918646,-0.013894715957772925,0.01733663937860758,0.005224431042220273,-0.010317303056153368,-0.13741099185924197,-0.003878403757536332,-0.030590930218256592,-0.008792223722254069,0.0019539696850622936,-0.007479632152379517,-0.029242993362549114,-0.02387135981277437,0.021846376648828582,-0.05826808307071031,0.00019395829280463247,0.057614417204062324,0.015150731664393562,0.00825988878455156,0.017728418725794937,0.009028666742709043,0.003885303805289234,-0.011138437613912942,0.010253049753112668,0.01218080460788482,0.020737996798181318,0.007403862005747379,-0.03330315142330753,0.008202620815249584,-0.008916952426693952,0.0018534995258902161,0.016552841208331094,0.004449248663274668,-0.026860492048011057,0.0015717269145302107,0.0030196489153873993,-0.0003977291228831768,0.022367249416718627,-0.04058160161507501,-0.022791272163030264,-0.005104002063476328,-0.013945965326021218,-0.005952573189171354,-0.03374339136195488,-0.010478052003909786,-0.018357175565627794,0.024262892596812666,-0.010352915520055997,0.004576074605877353,0.008566841547565592,0.0016967421676360955,0.03243168074292958,-0.0070771955049480605,0.005404794325734851,0.0037958903726275322,-0.0068716697568778655,0.0011996641922105858,0.010420590074357748,0.008913655834798362,0.012203324270666085,-0.005871938924270028,0.009803622712649616,0.021118386310681032,-0.010513438340266966,-0.005160788638567107,-0.04590735133845974,-0.0018737496628913825,-0.01246849510818393,-0.010669066550052973,0.028025055654708263]
p_embedding = matrix - p_matrix
n_embedding = matrix - n_matrix
return np.concatenate([p_embedding, n_embedding], axis=-1 )<load_pretrained>
%%time
grid_pipe_lasso.fit(X_train,y_train )
def make_token(n_tmpsize):
print_('Read file.')
 train = read_competision_file(train=True)
 test = read_competision_file(train=False)
 train_size = len(train)
 test_size = len(test)
 all_text = list(train['comment_text'])+ list(test['comment_text'])
 del train, test
gc.collect()
lemma_simple = {"\u00c1": "A", "\u1d8f": "a", "\u00e1": "a", "\u00c0": "A", "\u00e0": "a", "\u00c2": "A", "\u00e2": "a", "\u01cd": "A", "\u01ce": "a", "\u0102": "A", "\u0103": "a", "\u00c3": "A", "\u00e3": "a", "\u1ea2": "A", "\u1ea3": "a", "\u0226": "A", "\u0227": "a", "\u1ea0": "A", "\u1ea1": "a", "\u00c4": "A", "\u00e4": "a", "\u00c5": "A", "\u00e5": "a", "\u1e00": "A", "\u1e01": "a", "\u0100": "A", "\u0101": "a", "\u0104": "A", "\u0105": "a", "\u023a": "A", "\u2c65": "a", "\u0200": "A", "\u0201": "a", "\u1ea4": "A", "\u1ea5": "a", "\u1ea6": "A", "\u1ea7": "a", "\u1eaa": "A", "\u1eab": "a", "\u1ea8": "A", "\u1ea9": "a", "\u1eac": "A", "\u1ead": "a", "\u1eae": "A", "\u1eaf": "a", "\u1eb0": "A", "\u1eb1": "a", "\u1eb4": "A", "\u1eb5": "a", "\u1eb2": "A", "\u1eb3": "a", "\u1eb6": "A", "\u1eb7": "a", "\u01fa": "A", "\u01fb": "a", "\u01e0": "A", "\u01e1": "a", "\u01de": "A", "\u01df": "a", "\u0202": "A", "\u0203": "a", "\u2c6d": "A", "\u0251": "a", "\uff21": "A", "\uff41": "a", "\u1e02": "B", "\u1e03": "b", "\u1e04": "B", "\u1e05": "b", "\u1e06": "B", "\u1e07": "b", "\u0243": "B", "\u0180": "b", "\u0181": "B", "\u0253": "b", "\u0182": "B", "\u0183": "b", "\u1d6c": "B", "\u1d80": "b", "\u0299": "B", "\uff22": "b", "\uff42": "B", "\u0238": "d", "\u2114": "B", "\u0106": "C", "\u0107": "c", "\u0108": "C", "\u0109": "c", "\u010c": "C", "\u010d": "c", "\u010a": "C", "\u010b": "c", "\u0304": "c", "\u00c7": "C", "\u00e7": "c", "\u1e08": "C", "\u1e09": "c", "\u023b": "C", "\u023c": "c", "\u0187": "C", "\u0188": "c", "\u0255": "C", "\u1d04": "c", "\uff23": "C", "\uff43": "c", "\u010e": "D", "\u010f": "d", "\u1e0a": "D", "\u1e0b": "d", "\u1e10": "D", "\u1e11": "d", "\u1e0c": "D", "\u1e0d": "d", "\u1e12": "D", "\u1e13": "d", "\u1e0e": "D", "\u1e0f": "d", "\u0110": "D", "\u0111": "d", "\u0326": "d", "\u0189": "D", "\u0256": "d", "\u018a": "D", "\u0257": "d", "\u018b": "D", "\u018c": "d", "\u1d6d": "D", "\u1d81": "d", "\u1d91": "D", "\u0221": "d", "\u1d05": "D", "\uff24": "d", "\uff44": "D", "\u00c9": "E", "\u00e9": "e", "\u00c8": "E", "\u00e8": "e", "\u00ca": "E", "\u00ea": "e", "\u1e18": "E", "\u1e19": "e", "\u011a": "E", "\u011b": "e", "\u0114": "E", "\u0115": "e", "\u1ebc": "E", "\u1ebd": "e", "\u1e1a": "E", "\u1e1b": "e", "\u1eba": "E", "\u1ebb": "e", "\u0116": "E", "\u0117": "e", "\u00cb": "E", "\u00eb": "e", "\u0112": "E", "\u0113": "e", "\u0228": "E", "\u0229": "e", "\u0118": "E", "\u0119": "e", "\u1d92": "E", "\u0246": "e", "\u0247": "E", "\u0204": "e", "\u0205": "E", "\u1ebe": "e", "\u1ebf": "E", "\u1ec0": "e", "\u1ec1": "E", "\u1ec4": "e", "\u1ec5": "E", "\u1ec2": "e", "\u1ec3": "E", "\u1e1c": "e", "\u1e1d": "E", "\u1e16": "e", "\u1e17": "E", "\u1e14": "e", "\u1e15": "E", "\u0206": "e", "\u0207": "E", "\u1eb8": "e", "\u1eb9": "E", "\u1ec6": "e", "\u1ec7": "E", "\u2c78": "e", "\u1d07": "E", "\uff25": "e", "\uff45": "E", "\u1e1e": "F", "\u1e1f": "f", "\u0191": "F", "\u0192": "f", "\u1d6e": "F", "\u1d82": "f", "\ua730": "F", "\uff26": "f", "\uff46": "F", "\u01f4": "G", "\u01f5": "g", "\u011e": "G", "\u011f": "g", "\u011c": "G", "\u011d": "g", "\u01e6": "G", "\u01e7": "g", "\u0120": "G", "\u0121": "g", "\u0122": "G", "\u0123": "g", "\u1e20": "G", "\u1e21": "g", "\u01e4": "G", "\u01e5": "g", "\u0193": "G", "\u0260": "g", "\u1d83": "G", "\u0262": "g", "\uff27": "G", "\uff47": "g", "\u0124": "H", "\u0125": "h", "\u021e": "H", "\u021f": "h", "\u1e26": "H", "\u1e27": "h", "\u1e22": "H", "\u1e23": "h", "\u1e28": "H", "\u1e29": "h", "\u1e24": "H", "\u1e25": "h", "\u1e2a": "H", "\u1e2b": "h", "\u1e96": "h", "\u0126": "H", "\u0127": "h", "\u2c67": "H", "\u2c68": "h", "\u029c": "H", "\u0266": "h", "\uff28": "H", "\uff48": "h", "\u00cd": "I", "\u00ed": "i", "\u00cc": "I", "\u00ec": "i", "\u012c": "I", "\u012d": "i", "\u00ce": "I", "\u00ee": "i", "\u01cf": "I", "\u01d0": "i", "\u00cf": "I", "\u00ef": "i", "\u1e2e": "I", "\u1e2f": "i", "\u0128": "I", "\u0129": "i", "\u012e": "I", "\u012f": "i", "\u012a": "I", "\u012b": "i", "\u1ec8": "I", "\u1ec9": "i", "\u0208": "I", "\u0209": "i", "\u020a": "I", "\u020b": "i", "\u1eca": "I", "\u1ecb": "i", "\u1e2c": "I", "\u1e2d": "i", "\u0197": "I", "\u0268": "i", "\u1d7b": "I", "\u0130": "I", "\u1d96": "i", "\u026a": "I", "\u0131": "i", "\uff29": "I", "\uff49": "i", "\u0134": "J", "\u0135": "j", "\u0248": "J", "\u0249": "j", "\u01f0": "j", "\u0237": "J", "\u03f3": "j", "\u029d": "J", "\u025f": "j", "\u0284": "J", "\u1d0a": "j", "\uff2a": "J", "\uff4a": "j", "\u1e30": "K", "\u1e31": "k", "\u01e8": "K", "\u01e9": "k", "\u0136": "K", "\u0137": "k", "\u1e32": "K", "\u1e33": "k", "\u1e34": "K", "\u1e35": "k", "\u0198": "K", "\u0199": "k", "\u2c69": "K", "\u2c6a": "k", "\u1d84": "k", "\ua740": "K", "\ua741": "k", "\u1d0b": "K", "\uff2b": "k", "\uff4b": "K", "\u0139": "L", "\u013a": "l", "\u013d": "L", "\u013e": "l", "\u013b": "L", "\u013c": "l", "\u1e36": "L", "\u1e37": "l", "\u1e38": "L", "\u1e39": "l", "\u1e3c": "L", "\u1e3d": "l", "\u1e3a": "L", "\u1e3b": "l", "\u0141": "L", "\u0142": "l", "\u013f": "L", "\u0140": "l", "\u023d": "L", "\u019a": "l", "\u2c60": "L", "\u2c61": "l", "\u2c62": "L", "\u026b": "l", "\u026c": "L", "\u1d85": "l", "\u026d": "L", "\u0234": "l", "\u029f": "L", "\uff2c": "l", "\uff4c": "L", "\u1e3e": "M", "\u1e3f": "m", "\u1e40": "M", "\u1e41": "m", "\u1e42": "M", "\u1e43": "m", "\u1d6f": "M", "\u1d86": "m", "\u2c6e": "M", "\u0271": "m", "\u1d0d": "M", "\uff2d": "m", "\uff4d": "M", "\u0143": "N", "\u0144": "n", "\u01f8": "N", "\u01f9": "n", "\u0147": "N", "\u0148": "n", "\u00d1": "N", "\u00f1": "n", "\u1e44": "N", "\u1e45": "n", "\u0145": "N", "\u0146": "n", "\u1e46": "N", "\u1e47": "n", "\u1e4a": "N", "\u1e4b": "n", "\u1e48": "N", "\u1e49": "n", "\u019d": "N", "\u0272": "n", "\u0220": "N", "\u019e": "n", "\u1d70": "N", "\u1d87": "n", "\u0274": "N", "\u0273": "n", "\uff2e": "N", "\uff4e": "n", "\u014a": "N", "\u014b": "n", "\u0235": "n", "\u00d3": "O", "\u00f3": "o", "\u00d2": "O", "\u00f2": "o", "\u014e": "O", "\u014f": "o", "\u00d4": "O", "\u00f4": "o", "\u1ed0": "O", "\u1ed1": "o", "\u1ed2": "O", "\u1ed3": "o", "\u1ed6": "O", "\u1ed7": "o", "\u1ed4": "O", "\u1ed5": "o", "\u01d1": "O", "\u01d2": "o", "\u00d6": "O", "\u00f6": "o", "\u022a": "O", "\u022b": "o", "\u0150": "O", "\u0151": "o", "\u00d5": "O", "\u00f5": "o", "\u1e4c": "O", "\u1e4d": "o", "\u1e4e": "O", "\u1e4f": "o", "\u022c": "O", "\u022d": "o", "\u022e": "O", "\u022f": "o", "\u0230": "O", "\u0231": "o", "\u00d8": "O", "\u00f8": "o", "\u01fe": "O", "\u01ff": "o", "\u01ea": "O", "\u01eb": "o", "\u01ec": "O", "\u01ed": "o", "\u014c": "O", "\u014d": "O", "\u1e52": "O", "\u1e53": "o", "\u0302": "o", "\u1e50": "O", "\u1e51": "o", "\u1ece": "O", "\u1ecf": "o", "\u020c": "O", "\u020d": "o", "\u020e": "O", "\u020f": "o", "\u01a0": "O", "\u01a1": "o", "\u1eda": "O", "\u1edb": "o", "\u1edc": "O", "\u1edd": "o", "\u1ee0": "O", "\u1ee1": "o", "\u1ede": "O", "\u1edf": "o", "\u1ee2": "O", "\u1ee3": "o", "\u1ecc": "O", "\u1ecd": "o", "\u1ed8": "O", "\u1ed9": "o", "\u019f": "O", "\u0275": "o", "\u2c7a": "O", "\u1d0f": "o", "\uff2f": "O", "\uff4f": "o", "\u1e54": "P", "\u1e55": "p", "\u1e56": "P", "\u1e57": "p", "\u2c63": "P", "\u1d7d": "p", "\u01a4": "P", "\u01a5": "p", "\u1d71": "P", "\u1d88": "p", "\u1d18": "P", "\uff30": "p", "\uff50": "P", "\u024a": "Q", "\u024b": "q", "\uff31": "Q", "\uff51": "q", "\u0239": "Q", "\u02a0": "q", "\u0154": "R", "\u0155": "r", "\u0158": "R", "\u0159": "r", "\u1e58": "R", "\u1e59": "r", "\u0156": "R", "\u0157": "r", "\u0210": "R", "\u0211": "r", "\u0212": "R", "\u0213": "r", "\u1e5a": "R", "\u1e5b": "r", "\u1e5c": "R", "\u1e5d": "r", "\u1e5e": "R", "\u1e5f": "r", "\u024c": "R", "\u024d": "r", "\u2c64": "R", "\u027d": "r", "\u1d72": "R", "\u1d89": "r", "\u027c": "R", "\u027e": "r", "\u1d73": "R", "\u0280": "r", "\uff32": "R", "\uff52": "r", "\u015a": "S", "\u015b": "s", "\u1e64": "S", "\u1e65": "s", "\u015c": "S", "\u015d": "s", "\u0160": "S", "\u0161": "s", "\u1e66": "S", "\u1e67": "s", "\u1e60": "S", "\u1e61": "s", "\u015e": "S", "\u015f": "s", "\u1e62": "S", "\u1e63": "s", "\u1e68": "S", "\u1e69": "s", "\u0218": "S", "\u0219": "s", "\u1d74": "s", "\u1d8a": "s", "\u0282": "s", "\u023f": "s", "\ua731": "s", "\uff33": "S", "\uff53": "s", "\u0164": "T", "\u0165": "t", "\u1e6a": "T", "\u1e6b": "t", "\u0162": "T", "\u0163": "t", "\u1e6c": "T", "\u1e6d": "t", "\u021a": "T", "\u021b": "t", "\u1e70": "T", "\u1e71": "t", "\u1e6e": "T", "\u1e6f": "t", "\u0166": "T", "\u0167": "t", "\u023e": "T", "\u2c66": "t", "\u01ac": "T", "\u01ad": "t", "\u01ae": "T", "\u0288": "t", "\u1e97": "t", "\u1d75": "T", "\u01ab": "t", "\u0236": "T", "\uff34": "t", "\u1d1b": "T", "\uff54": "T", "\u00da": "U", "\u00fa": "u", "\u00d9": "U", "\u00f9": "u", "\u016c": "U", "\u016d": "u", "\u00db": "U", "\u00fb": "u", "\u01d3": "U", "\u01d4": "u", "\u016e": "U", "\u016f": "u", "\u00dc": "U", "\u00fc": "u", "\u01d7": "U", "\u01d8": "u", "\u01db": "U", "\u01dc": "u", "\u01d9": "U", "\u01da": "u", "\u01d5": "U", "\u01d6": "u", "\u0170": "U", "\u0171": "u", "\u0168": "U", "\u0169": "u", "\u1e78": "U", "\u1e79": "u", "\u0172": "U", "\u0173": "u", "\u016a": "U", "\u016b": "u", "\u1e7a": "U", "\u1e7b": "u", "\u1ee6": "U", "\u1ee7": "u", "\u0214": "U", "\u0215": "u", "\u0216": "U", "\u0217": "u", "\u01af": "U", "\u01b0": "u", "\u1ee8": "U", "\u1ee9": "u", "\u1eea": "U", "\u1eeb": "u", "\u1eee": "U", "\u1eef": "u", "\u1eec": "U", "\u1eed": "u", "\u1ef0": "U", "\u1ef1": "u", "\u1ee4": "U", "\u1ee5": "u", "\u1e72": "U", "\u1e73": "u", "\u1e76": "U", "\u1e77": "u", "\u1e74": "U", "\u1e75": "u", "\u0244": "U", "\u0289": "u", "\u1d7e": "U", "\u1d99": "u", "\u1d1c": "U", "\uff35": "u", "\uff55": "U", "\u1e7c": "V", "\u1e7d": "v", "\u1e7e": "V", "\u1e7f": "v", "\u01b2": "V", "\u028b": "v", "\u1d8c": "v", "\u2c71": "V", "\u2c74": "v", "\uff36": "V", "\u1d20": "v", "\uff56": "v", "\u1e82": "W", "\u1e83": "w", "\u1e80": "W", "\u1e81": "w", "\u0174": "W", "\u0175": "w", "\u1e84": "W", "\u1e85": "w", "\u1e86": "W", "\u1e87": "w", "\u1e88": "W", "\u1e89": "w", "\u1e98": "w", "\u2c72": "W", "\u2c73": "w", "\uff37": "W", "\uff57": "w", "\u1e8c": "X", "\u1e8d": "x", "\u1e8a": "X", "\u1e8b": "x", "\u1d8d": "X", "\uff38": "x", "\uff58": "X", "\u00dd": "Y", "\u00fd": "y", "\u1ef2": "Y", "\u1ef3": "y", "\u0176": "Y", "\u0177": "y", "\u0178": "Y", "\u1e99": "y", "\u00ff": "y", "\u1ef8": "Y", "\u1ef9": "y", "\u1e8e": "Y", "\u1e8f": "y", "\u0232": "Y", "\u0233": "y", "\u1ef6": "Y", "\u1ef7": "y", "\u1ef4": "Y", "\u1ef5": "y", "\u024e": "Y", "\u024f": "y", "\u01b3": "Y", "\u01b4": "y", "\uff39": "Y", "\u028f": "y", "\uff59": "y", "\u0179": "Z", "\u017a": "z", "\u1e90": "Z", "\u1e91": "z", "\u017d": "Z", "\u017e": "z", "\u017b": "Z", "\u017c": "z", "\u1e92": "Z", "\u1e93": "z", "\u1e94": "Z", "\u1e95": "z", "\u01b5": "Z", "\u01b6": "z", "\u0224": "Z", "\u0225": "z", "\u2c6b": "Z", "\u2c6c": "z", "\u1d76": "Z", "\u1d8e": "z", "\u0290": "Z", "\u0291": "z", "\u0240": "Z", "\u1d22": "z", "\uff3a": "Z", "\uff5a": "z"}
tokenizer = TokenVectorizer(vector_dict='.. /input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl', lemma_dict=lemma_simple, spell_collector={}, num_process=3)
 print_("Preprocess.")
 all_text = tokenizer.preprocess(all_text)
 l = train_size//n_tmpsize
for n in range(n_tmpsize):
print_("Tokenize %d/%d."%(n+1,n_tmpsize+1))
fp = l * n
ep = fp + l
if n == n_tmpsize-1:
ep = train_size
tokens = tokenizer.tokenize(all_text[fp:ep],maxlen=275,pad_sequence=False)
 with open('temporary_tokens_%d.pickle'%(n+1), mode='wb')as f:
pickle.dump(tokens, f)
 del tokens
gc.collect()
print_("Tokenize %d/%d."%(n_tmpsize+1,n_tmpsize+1))
tokens = tokenizer.tokenize(all_text[train_size:],maxlen=275,pad_sequence=False)
 with open('temporary_tokens_test.pickle', mode='wb')as f:
pickle.dump(tokens, f)
 del tokens
gc.collect()
print_("Vectorize.")
 return tokenizer.vectorize() , test_size<categorify>
grid_pipe_lasso.best_params_
def make_temp(n_tmpsize):
crawl_matrix, test_size = make_token(n_tmpsize)
 gc.collect()
print_('Read file.')
 train = read_competision_file(train=True)
 y_train_o =(train['target'].values>=0.5 ).astype(np.int)
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
embedding_matrix = make_isolated_embedding(crawl_matrix)
 return train, embedding_matrix, y_train_o, y_aux_train, test_size<define_search_space>
grid_pipe_lasso.best_score_
def get_weight(n_tmpsize):
train, embedding_matrix, y_train_o, y_aux_train, test_size = make_temp(n_tmpsize)
 gc.collect()
weight_factor = [0.25,0.0,0.0,0.0,0.96,0.34]
identity_factor_1 = [0.0,0.0,0.0,0.05,0.0,0.0,-0.05,0.0,0.0]
identity_factor_2 = [-0.05,0.0,0.05,-0.05,0.0,0.0,0.0,0.0,0.0]
aux_impact_factor = [1.5,1.5,1.2,1.0,1.2,1.2]
aux_identity_factor = [0.8,0.75,1.15,1.2]
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
index_subgroup, index_bpsn, index_bnsp = dict() , dict() , dict()
for col in identity_columns:
index_subgroup[col] =(train[col].fillna(0 ).values>=0.5 ).astype(bool)
 index_bpsn[col] =(((( train['target'].values<0.5 ).astype(bool ).astype(np.int)+(train[col].fillna(0 ).values>=0.5 ).astype(bool ).astype(np.int)) > 1 ).astype(bool)) +(((( train['target'].values>=0.5 ).astype(bool ).astype(np.int)+(train[col].fillna(0 ).values<0.5 ).astype(bool ).astype(np.int)) > 1 ).astype(bool))
index_bnsp[col] =(((( train['target'].values>=0.5 ).astype(bool ).astype(np.int)+(train[col].fillna(0 ).values>=0.5 ).astype(bool ).astype(np.int)) > 1 ).astype(bool)) +(((( train['target'].values<0.5 ).astype(bool ).astype(np.int)+(train[col].fillna(0 ).values<0.5 ).astype(bool ).astype(np.int)) > 1 ).astype(bool))
weights = np.ones(( len(train),)) * weight_factor[0]
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)* weight_factor[1]
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)* weight_factor[2]
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)* weight_factor[3]
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)* weight_factor[4]
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)* weight_factor[5]
index_id1, index_id2 = dict() , dict()
for col in identity_columns:
index_id1[col] =(((train[col].fillna(0 ).values>=0.5 ).astype(bool ).astype(np.int)+(train['target'].values>=0.5 ).astype(bool ).astype(np.int)) > 1 ).astype(bool)
 index_id2[col] =(((train[col].fillna(0 ).values>=0.5 ).astype(bool ).astype(np.int)+(train['target'].values<0.5 ).astype(bool ).astype(np.int)) > 1 ).astype(bool)
 for col,id1 in zip(identity_columns, identity_factor_1):
weights[index_id1[col]] += id1
for col,id2 in zip(identity_columns, identity_factor_2):
weights[index_id2[col]] += id2
loss_weight = 1.0 / weights.mean()
weights_aux = np.ones(( len(train),))
weights_aux[(train['target'].values>=0.5 ).astype(np.int)+(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)> 1] = aux_identity_factor[0]
weights_aux[(train['target'].values>=0.5 ).astype(np.int)+(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)> 1] = aux_identity_factor[1]
weights_aux[(train['target'].values<0.5 ).astype(np.int)+(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)> 1] = aux_identity_factor[2]
weights_aux[(train['target'].values<0.5 ).astype(np.int)+(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)> 1] = aux_identity_factor[3]
y_train = np.vstack([y_train_o,weights,weights_aux] ).T
y_train = np.hstack([y_train, y_aux_train])
 return embedding_matrix, y_train, loss_weight, aux_impact_factor, test_size<normalization>
print(np.sqrt(mean_squared_log_error(y_test,grid_pipe_lasso.predict(X_test))))
class GaussianNoise(nn.Module):
def __init__(self, stddev):
super(GaussianNoise, self ).__init__()
self.stddev = stddev
def forward(self, x):
if self.training:
noise = torch.empty_like(x)
 noise.normal_(0, self.stddev)
 return x + noise
else:
return x
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix):
super(NeuralNet, self ).__init__()
self.embedding = nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1])
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm = nn.LSTM(embedding_matrix.shape[1], 128, num_layers=2, bidirectional=True, batch_first=True)
 self.noise = GaussianNoise(0.2)
 self.bn = nn.BatchNorm1d(512, momentum=0.5)
 self.hidden1 = nn.Linear(512, 512)
 self.hidden2 = nn.Linear(512, 512)
 self.linear_out = nn.Linear(512, 7)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_layer, _ = self.lstm(h_embedding)
 avg_pool = torch.mean(h_layer, 1)
 max_pool, _ = torch.max(h_layer, 1)
 h_conc = torch.cat(( avg_pool, max_pool), 1)
 h_conc = self.noise(h_conc)
 h_conc = self.bn(h_conc)
 h_conc_linear1 = F.relu(self.hidden1(h_conc))
h_conc_linear2 = F.relu(self.hidden2(h_conc_linear1))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
out = self.linear_out(hidden)
 return out<define_variables>
from sklearn.ensemble import GradientBoostingRegressor
def get_batchs(file, batchsize=256):
with open(file, mode='rb')as f:
tokens = pickle.load(f)
 for sp in range(0,len(tokens),batchsize):
ep = min(len(tokens),sp+batchsize)
 tok = tokens[sp:ep]
nwd = np.max([len(t)for t in tok])
 batch = []
for i in range(ep-sp):
l = len(tok[i])
 batch.append(( [0]*(nwd-l)) +tok[i])
 yield batch<compute_test_metric>
gbdt_model = GradientBoostingRegressor(learning_rate=0.05, min_samples_leaf=5,
min_samples_split=10, max_depth=4, n_estimators=3000 )
def sigmoid(x):
return 1 /(1 + np.exp(-x))<categorify>
pipeline_gbt = Pipeline([ 
('featureunion',combined_pipeline), 
('dictifier',FunctionTransformer(func=return_dict,validate=False)) , 
('vectorizer',DictVectorizer(sort=False,sparse=False)) , 
('reg_lasso',gbdt_model)
 ] )
class DataLoader2:
def __init__(self, X, Y, batch_size):
self.X = X
self.Y = torch.tensor(Y ).cuda()
self.dataset = self
self.batch_size = batch_size
self.batch_pos = 0
def __next__(self):
x = torch.tensor(self.X.__next__() ).cuda()
y = self.Y[self.batch_pos:self.batch_pos+len(x)]
self.batch_pos += len(x)
 return(x,y)
 def __iter__(self):
return self
def __len__(self):
return len(self.Y)// self.batch_size +(1 if len(self.Y)%self.batch_size != 0 else 0)
 class DataBunch2:
def __init__(self, X, Y, batch_size):
self.path = '.'
self.device = torch.device('cuda')
 self.train_dl = DataLoader2(X, Y, batch_size)
 self.empty_val = True<train_on_grid>
%%time
pipeline_gbt.fit(X_train,y_train )
def weighted_train(num_models=1, n_tmpsize=8, checkpoint_weights=[1,2,4,8], batch_size=256):
embedding_matrix, y_train, loss_weight, aux_impact_factor, test_size = get_weight(n_tmpsize)
 gc.collect()
def custom_loss_aux(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1].double() ,targets[:,:1])
 bce_loss_aux_1 = nn.BCEWithLogitsLoss(weight=targets[:,2:3] )(data[:,1:2].double() ,targets[:,3:4])
 bce_loss_aux_2 = nn.BCEWithLogitsLoss(weight=targets[:,2:3] )(data[:,2:3].double() ,targets[:,4:5])
 bce_loss_aux_3 = nn.BCEWithLogitsLoss(weight=targets[:,2:3] )(data[:,3:4].double() ,targets[:,5:6])
 bce_loss_aux_4 = nn.BCEWithLogitsLoss(weight=targets[:,2:3] )(data[:,4:5].double() ,targets[:,6:7])
 bce_loss_aux_5 = nn.BCEWithLogitsLoss(weight=targets[:,2:3] )(data[:,5:6].double() ,targets[:,7:8])
 bce_loss_aux_6 = nn.BCEWithLogitsLoss(weight=targets[:,2:3] )(data[:,6:7].double() ,targets[:,8:9])
 return(bce_loss_1 * loss_weight)+(bce_loss_aux_1 * aux_impact_factor[0])+(bce_loss_aux_2 * aux_impact_factor[1])+(bce_loss_aux_3 * aux_impact_factor[2])+(bce_loss_aux_4 * aux_impact_factor[3])+(bce_loss_aux_5 * aux_impact_factor[4])+(bce_loss_aux_6 * aux_impact_factor[5])
 checkpoint_predictions = []
model = NeuralNet(embedding_matrix)
 del embedding_matrix
gc.collect()
optimizer = Nadam(filter(lambda p: p.requires_grad, model.parameters()))
weights = []
checkpoint_predictions = []
for n in range(num_models):
before_epoch_weight = 0
for global_epoch, epoch_weight in enumerate(checkpoint_weights):
print_("Model
l = len(y_train)// n_tmpsize
y_start = [l * nt for nt in range(n_tmpsize)]
y_end = [l *(nt+1)for nt in range(n_tmpsize-1)] + [len(y_train)]
x_files = ['temporary_tokens_%d.pickle'%(nt+1)for nt in range(n_tmpsize)]
for global_iter, p in enumerate(np.random.permutation(n_tmpsize)) :
databunch = DataBunch2(X=get_batchs(x_files[p]), Y=y_train[y_start[p]:y_end[p]], batch_size=batch_size)
 learn = Learner(databunch, model, loss_func=custom_loss_aux, opt_func=Nadam)
 learn.fit(1)
 test_preds = np.zeros(( test_size, 7))
batch_pos = 0
with torch.no_grad() :
for x_batch in get_batchs('temporary_tokens_test.pickle'):
X = torch.tensor(x_batch ).cuda()
test_preds[batch_pos:batch_pos+len(X)] = sigmoid(model(X ).detach().cpu().numpy())
 batch_pos += len(X)
 checkpoint_predictions.append(test_preds[:,0].flatten())
 weights.append(before_epoch_weight +(epoch_weight - before_epoch_weight)*(( global_iter + 1)/ n_tmpsize))
before_epoch_weight = epoch_weight
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 return predictions<save_to_csv>
print(np.sqrt(mean_squared_log_error(y_test,pipeline_gbt.predict(X_test))))
def make_lstm_submission(output_file):
predictions = weighted_train()
df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df['prediction'] = predictions
df.to_csv(output_file, index=False )<save_to_csv>
from sklearn.ensemble import VotingRegressor
make_bert_submission('bert.csv')
 make_lstm_submission('lstm.csv')
 df1 = pd.read_csv('bert.csv')
 df2 = pd.read_csv('lstm.csv')
 df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df['prediction'] =(df1.prediction + df2.prediction)/ 2.
 df.to_csv('submission.csv', index=False )<define_variables>
xgb_final = xgb.XGBRegressor(objective="reg:squarederror",
n_estimators=200,
max_depth=5,
gamma=0.5,
colsample_bytree=0.3 )
BASE_DIR= ".. /input/"<load_from_csv>
lgm_final = lgb.LGBMRegressor(reg_lambda=0.1,
n_estimators=100,
learning_rate=0.1,
colsample_bytree=0.2 )
salesData= pd.read_csv(f"{BASE_DIR}sales_train.csv")
 items= pd.read_csv(f"{BASE_DIR}items.csv")
 testDF= pd.read_csv(f"{BASE_DIR}test.csv")
 itemCategories= pd.read_csv(f"{BASE_DIR}item_categories.csv")
 shops= pd.read_csv(f"{BASE_DIR}shops.csv")
 salesData.drop(['date'],axis=1,inplace=True )<count_unique_values>
en_final = ElasticNet(l1_ratio=0.8,alpha=0.5 )
uniqueTestItems = testDF['item_id'].nunique()
print("Number of unique test items = "+str(uniqueTestItems))
uniqueTrainItems = salesData['item_id'].nunique()
print("Number of unique train items = "+str(uniqueTrainItems))<count_unique_values>
lasso_final = Lasso(alpha=0.001 )
uniqueTestShops = testDF['shop_id'].nunique()
print("Number of unique test shops = "+str(uniqueTestShops))
uniqueTrainShops = salesData['shop_id'].nunique()
print("Number of unique train shops = "+str(uniqueTrainShops))<count_unique_values>
gb_final = GradientBoostingRegressor(learning_rate=0.05, min_samples_leaf=5,
min_samples_split=10, max_depth=4, n_estimators=3000 )
uniqueTrainItems = salesData['item_id'].unique()
itemsNotInTrainDF = testDF[ ~testDF['item_id'].isin(uniqueTrainItems)]
itemsInTrainDF = testDF[ testDF['item_id'].isin(uniqueTrainItems)]
itemsNotInTrainDF.shape[0]/itemsInTrainDF.shape[0]<feature_engineering>
final_regressor = VotingRegressor(estimators=[ 
('xgb',xgb_final), 
('lgm',lgm_final), 
('elastic',en_final), 
('lasso',lasso_final), 
('gbreg',gb_final)
 ] )
tmpCat= itemCategories.copy()
tmpCat['isHyphenInName']= tmpCat['item_category_name'].apply(lambda x: 1 if '-' in x else 0)
 tmpCat['isHyphenInName'].sum() /tmpCat.shape[0]<count_unique_values>
pipeline_final = Pipeline([ 
('featureunion',combined_pipeline), 
('dictifier',FunctionTransformer(func=return_dict,validate=False)) , 
('vectorizer',DictVectorizer(sort=False)) , 
('regressor',final_regressor)
 ] )
tmpShops= shops.copy()
tmpShops['firstWordInName']= tmpShops['shop_name'].apply(lambda x: x.split(' ')[0])
 tmpShops['firstWordInName'].nunique() /tmpShops.shape[0]<filter>
X = train_data.drop(['SalePrice'],axis=1)
 y = train_data['SalePrice']
salesData= filteredTrainDataWithItemPair
salesData=salesData[(salesData['item_price']<50000)&(salesData['item_cnt_day']<500)]<groupby>
pipeline_final.fit(X,y )
targetDF= salesData.groupby(['item_id','shop_id','date_block_num'],as_index=False ).agg({'item_cnt_day':['sum'],'item_price':'mean'})
 targetDF.columns= ['item_id','shop_id','date_block_num','target','avg_price']<data_type_conversions>
final_preds = pipeline_final.predict(test_data )
testDF['date_block_num'] = 34
testDF['date_block_num'] = testDF['date_block_num'].astype(np.int8)
 testDF['shop_id'] = testDF['shop_id'].astype(np.int8)
 testDF['item_id'] = testDF['item_id'].astype(np.int16)
 cartesianDF = pd.concat([cartesianDF, testDF], ignore_index=True, sort=False, keys=['shop_id','item_id','date_block_num'] )<merge>
submission = pd.DataFrame({ 'Id': ID,
'SalePrice': final_preds })
 submission.to_csv(path_or_buf ="Advanced_Housing_Regression.csv", index=False)