Kaggle competition: Rossmann Store Sales
<SOS> metric: RootMeanSquarePercentageError Kaggle data source: rossmann-store-sales<predict_on_test>
warnings.filterwarnings('ignore' )
eng1 = model.predict(test1_dataset, verbose=1)
 eng2 = model.predict(test2_dataset, verbose=1 )<save_to_csv>
df_train = pd.read_csv('.. /input/rossmann-store-sales/train.csv' )
sub.to_csv('submission.csv', index=False )<save_to_csv>
df_train.isnull().sum()
sub.to_csv('submission.csv', index=False )<choose_model_class>
df_train['DayOfWeek'].value_counts()
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=3e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC() ])
 return model<compute_test_metric>
len(df_train['Store'].unique() )
def roc_auc(predictions,target):

 fpr, tpr, thresholds = metrics.roc_curve(target, predictions)
 roc_auc = metrics.auc(fpr, tpr)
 return roc_auc<import_modules>
print('Open:
', df_train['Open'].value_counts() , '
')
 print('Promo
', df_train['Promo'].value_counts() , '
')
 print('State Holiday
', df_train['StateHoliday'].value_counts() , '
')
 print('School Holiday
', df_train['SchoolHoliday'].value_counts() )
import numpy as np
import pandas as pd
from tqdm import tqdm
import os
from sklearn import metrics
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from tokenizers import BertWordPieceTokenizer<set_options>
df_train[df_train['StateHoliday'] == 0]['StateHoliday'].value_counts()
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
df_train[df_train['StateHoliday'] == '0']['StateHoliday'].value_counts()
DATA_PATH = ".. /input/jigsaw-multilingual-toxic-comment-classification"<load_from_csv>
df_train['StateHoliday'] = df_train['StateHoliday'].apply(lambda x: 0 if x == '0' else x)
 df_train['StateHoliday'].value_counts()
wiki_toxic_comment_data = "jigsaw-toxic-comment-train.csv"
toxic_unintended_data = "jigsaw-unintended-bias-train.csv"
valid_data = "validation.csv"
test_data = "test.csv"
sub_sample = "sample_submission.csv"
wiki_toxic_comment = pd.read_csv(os.path.join(DATA_PATH, wiki_toxic_comment_data))
toxic_unintended = pd.read_csv(os.path.join(DATA_PATH, toxic_unintended_data))
valid = pd.read_csv(os.path.join(DATA_PATH, valid_data))
test = pd.read_csv(os.path.join(DATA_PATH, test_data))
sub = pd.read_csv(os.path.join(DATA_PATH, sub_sample)) 
<define_variables>
df_store = pd.read_csv('.. /input/rossmann-store-sales/store.csv')
 df_store.head()
EPOCHS = 2
BATCH_SIZE_PER_CORE = 64
BATCH_SIZE = BATCH_SIZE_PER_CORE * strategy.num_replicas_in_sync
MAX_LEN = 200<categorify>
df_store.isnull().sum()
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<load_pretrained>
df_store['StoreType'].value_counts()
tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-multilingual-cased')
 tokenizer.save_pretrained('.')
 fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)
 fast_tokenizer<prepare_x_and_y>
df_store['Assortment'].value_counts()
%%time
x_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 y_train = train1.toxic.values
y_valid = valid.toxic.values<feature_engineering>
df_store['StoreType'] = df_store['StoreType'].apply(lambda x: 1 if x == 'a' else(2 if x == 'b' else(3 if x == 'c' else 4)))
 df_store['StoreType'].value_counts()
AUTO = tf.data.experimental.AUTOTUNE<split>
df_store['Assortment'] = df_store['Assortment'].apply(lambda x: 1 if x == 'a' else(2 if x == 'b' else 3))
df_store['Assortment'].value_counts()
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(len(x_train)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<load_pretrained>
df_store['CompetitionDistance'] = df_store['CompetitionDistance'].fillna(max(df_store['CompetitionDistance']))
df_store.info()
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFBertModel 
.from_pretrained('bert-base-multilingual-cased')
)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
def mapping(features):
for feature in features:
temp_dict = {}
temp_dict = pd.Series(df_store[feature].values, index = df_store['Store'] ).to_dict()
df_train[feature] = df_train['Store'].map(temp_dict )
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
mapping(['StoreType', 'Assortment', 'CompetitionDistance'] )
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS*2
)<predict_on_test>
df_train[df_train['Sales'] == 0]
scores = model.predict(valid_dataset)
 print("Auc: %.2f%%" %(roc_auc(scores,y_valid)*100))<save_to_csv>
df_train[df_train['Open'] == 0]
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<define_variables>
df_train[df_train['Open'] == 0]['Sales'].value_counts()
EPOCHS = 2
BATCH_SIZE_PER_CORE = 32
BATCH_SIZE = BATCH_SIZE_PER_CORE * strategy.num_replicas_in_sync
MAX_LEN = 200<categorify>
df_train['Open'].value_counts()
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<load_pretrained>
print('Promo
', df_train['Promo'].value_counts() , '
')
 print('State Holiday
', df_train['StateHoliday'].value_counts() , '
')
 print('School Holiday
', df_train['SchoolHoliday'].value_counts() )
tokenizer = transformers.AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large' )<prepare_x_and_y>
df_train['StateHoliday'] = df_train['StateHoliday'].apply(lambda x: 1 if x == 'a' else(2 if x == 'b' else(3 if x == 'c' else x)))
 df_train['StateHoliday'].value_counts()
%%time
x_train = regular_encode(train1.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train1.toxic.values
y_valid = valid.toxic.values<feature_engineering>
df_train['DayOfYear'] = df_train['Date'].map(lambda x: datetime.datetime.strptime(str(x),'%Y-%m-%d' ).timetuple().tm_yday)
 df_train.head(10 )
AUTO = tf.data.experimental.AUTOTUNE<split>
df_train['Date'] = pd.to_datetime(df_train['Date'], format = '%Y-%m-%d' )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(len(x_train)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<train_model>
df_train['Year'] = df_train['Date'].map(lambda x: x.year)
 df_train.head()
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
df_train.drop('Date', inplace = True, axis = 1)
 df_train.drop('Customers', inplace = True, axis = 1)
 df_train.shape
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS*2
)<predict_on_test>
X = df_train.drop('Sales', axis = 1)
 y = df_train['Sales']
scores = model.predict(valid_dataset)
 print("Auc: %.2f%%" %(roc_auc(scores,y_valid)*100))<save_to_csv>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size= 0.1, random_state = 53)
 X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state = 53 )
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<import_modules>
scaler = preprocessing.StandardScaler()
import os
import numpy as np
import pandas as pd
import sklearn
from sklearn.utils.class_weight import compute_class_weight
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.losses import BinaryCrossentropy
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<categorify>
X_train_scalled = scaler.fit_transform(X_train)
 X_val_scalled = scaler.transform(X_val)
 X_test_scalled = scaler.transform(X_test )
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<categorify>
linreg = LinearRegression()
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
linreg.fit(X_train_scalled, y_train )
loss = BinaryCrossentropy()<choose_model_class>
y_val_pred = linreg.predict(X_val_scalled )
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss=loss, metrics=['accuracy'])
 return model<set_options>
y_train_pred = linreg.predict(X_train_scalled )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
data = pd.DataFrame({'Actual':y_val, 'Predicted':y_val_pred})
 data
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'<load_pretrained>
r2_score(y_val, y_val_pred )
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
r2_score(y_train, y_train_pred )
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<prepare_x_and_y>
mae = metrics.mean_absolute_error(y_val, y_val_pred)
 mse = metrics.mean_squared_error(y_val, y_val_pred)
 rmse = np.sqrt(metrics.mean_absolute_error(y_val, y_val_pred))
print("Mean Absolute Error")
 print(mae)
 print()
print("Mean Squared Error")
 print(mse)
 print()
print("Root Mean Squared Error")
 print(rmse )
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
print('x_train.shape:', x_train.shape )<define_variables>
evaluation = pd.DataFrame()
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.shuffle(2048)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<load_pretrained>
def evaluation_df(method, mae, mse, rmse, evaluation):
temp_evaluation = pd.DataFrame({'Method':[method], 'MAE': [mae], 'MSE': [mse], 'RMSE': [rmse]})
 evaluation = pd.concat([evaluation, temp_evaluation])
 evaluation = evaluation[['Method', 'MAE', 'MSE', 'RMSE']]
return evaluation
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
evaluation = evaluation_df('Linear Regression', mae, mse, rmse, evaluation )
n_steps = x_train.shape[0] // BATCH_SIZE
class_weights = compute_class_weight(class_weight = 'balanced', classes = np.unique(train['toxic'].values), y = train['toxic'].values)
 train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
class_weight = class_weights
)<train_model>
!pip install xgboost
n_steps = x_valid.shape[0] // BATCH_SIZE
class_weights = compute_class_weight(class_weight = 'balanced', classes = np.unique(valid['toxic'].values), y = valid['toxic'].values)
 train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS,
class_weight = class_weights
)<predict_on_test>
%%time
xgbreg = xgb.XGBRegressor()
xgbreg.fit(X_train_scalled, y_train )
predictions = model.predict(test_dataset, verbose=1 )<categorify>
xgbreg.score(X_train_scalled, y_train )
predictions = predictions.round().astype(int)
 pseudo_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_test, predictions)) 
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<train_model>
y_train_pred = xgbreg.predict(X_train_scalled )
n_steps = x_test.shape[0] // BATCH_SIZE
model.fit(
 pseudo_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<save_to_csv>
r2_score(y_train, y_train_pred )
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<find_best_params>
y_val_pred = xgbreg.predict(X_val_scalled )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.cluster_spec().as_dict() ['worker'])
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()<load_from_url>
r2_score(y_val, y_val_pred )
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev 
<feature_engineering>
mae = metrics.mean_absolute_error(y_val, y_val_pred)
 mse = metrics.mean_squared_error(y_val, y_val_pred)
 rmse = np.sqrt(metrics.mean_absolute_error(y_val, y_val_pred))
print("Mean Absolute Error")
 print(mae)
 print()
print("Mean Squared Error")
 print(mse)
 print()
print("Root Mean Squared Error")
 print(rmse )
os.environ['XLA_USE_BF16'] = "1"
os.environ['XLA_TENSOR_ALLOCATOR_MAXSIZE'] = '100000000'
warnings.filterwarnings("ignore" )<define_variables>
evaluation = evaluation_df('Extreme Gradient Boosting', mae, mse, rmse, evaluation )
PATH = ""
MAX_LEN = 224<load_from_csv>
df_train = pd.read_csv(".. /input/jigsaw-public-baseline-train-data/train_data.csv", usecols=["comment_text", "toxic", "lang"])
 df_valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv', usecols=["comment_text", "toxic", "lang"])
 df_test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv' ).rename(columns={"content": "comment_text"} )<feature_engineering>
labels = np.char.add(df_train.toxic.values.astype(str), df_train.lang.values)
 df_train["label"] = labels
min_size = df_train.groupby("label" ).size().min()
print(min_size)
 upsample = 1
samples = []
for i in range(3):
print(i)
 sample = []
for l in df_train.label.unique() :
if l[0] == "1":
x = df_train[df_train["label"]==l].sample(min_size, replace=False, random_state=i)
 sample.append(x)
 sample.append(df_train[df_train["label"]==f"0{l[1:]}"].sample(min_size*upsample, replace=False, random_state=i))
sample = pd.concat(sample, axis=0 ).sample(frac=1)
 del sample["label"]
DATA_LENGTH = len(sample)
 samples.append(sample)
 del sample<drop_column>
%%time
xgbreg = xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.45, max_delta_step=0, max_depth=7,
min_child_weight=15, monotone_constraints='() ',
n_estimators=120, n_jobs=16, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
tree_method='exact', validate_parameters=1, verbosity=None)
 xgbreg.fit(X_train_scalled, y_train )
del df_train
df_train = samples<set_options>
xgbreg.score(X_train_scalled, y_train )
gc.collect()
!free -h<choose_model_class>
y_train_pred = xgbreg.predict(X_train_scalled )
class CustomRoberta(nn.Module):
def __init__(self):
super(CustomRoberta, self ).__init__()
self.num_labels = 2
self.roberta = transformers.XLMRobertaModel.from_pretrained("xlm-roberta-large", output_hidden_states=False, num_labels=1)
 self.dropout = nn.Dropout(p=0.2)
 self.ln = nn.LayerNorm(1024)
 self.classifier = nn.Linear(1024, self.num_labels)
 def forward(self,
input_ids=None,
attention_mask=None,
position_ids=None,
head_mask=None,
inputs_embeds=None):
o1, o2 = self.roberta(input_ids,
attention_mask=attention_mask,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds)
 x1 = torch.mean(o1, 1)
 x = x1
x = self.ln(x)
 x = self.dropout(x)
 logits = self.classifier(x)
 return logits<load_pretrained>
r2_score(y_train, y_train_pred )
tokenizer = transformers.XLMRobertaTokenizer.from_pretrained('xlm-roberta-large')
 mx = xmp.MpModelWrapper(CustomRoberta() )<categorify>
y_val_pred = xgbreg.predict(X_val_scalled )
class BERTDataset:
def __init__(self, df=None):
self.df = df
def __len__(self):
return len(self.df)
 def get_tokens(self, text):
encoded = tokenizer.encode_plus(
 text,
add_special_tokens=True,
max_length=MAX_LEN,
pad_to_max_length=True 
)
 return encoded['input_ids'], encoded['attention_mask']
def get_labels(self):
return LabelEncoder().fit_transform(np.char.add(self.df.toxic.values.astype(str), self.df.lang.values ).reshape(-1,1)).astype(np.int16)
 def __getitem__(self, item):
text = self.df.iloc[item]["comment_text"]
lang = self.df.iloc[item]["lang"]
encoded = self.get_tokens(text)
 targets = np.zeros(2)
 if "toxic" in self.df.columns:
targets[self.df.iloc[item]["toxic"]] = 1
return {
'ids': torch.tensor(encoded[0]),
'mask': torch.tensor(encoded[1]),
'targets': targets,
'index': item
}<create_dataframe>
r2_score(y_val, y_val_pred )
valid_dataset = BERTDataset(df_valid)
 test_dataset = BERTDataset(df_test )<set_options>
mae = metrics.mean_absolute_error(y_val, y_val_pred)
 mse = metrics.mean_squared_error(y_val, y_val_pred)
 rmse = np.sqrt(metrics.mean_absolute_error(y_val, y_val_pred))
print("Mean Absolute Error")
 print(mae)
 print()
print("Mean Squared Error")
 print(mse)
 print()
print("Root Mean Squared Error")
 print(rmse )
gc.collect()
!free -h<compute_train_metric>
evaluation = evaluation_df('Extreme Gradient Boosting Tuning 1', mae, mse, rmse, evaluation )
class RocAucMeter(object):
def __init__(self):
self.reset()
def reset(self):
self.y_true = np.array([0,1])
 self.y_pred = np.array([0.5,0.5])
 self.score = 0
def update(self, y_true, y_pred):
y_true = y_true.cpu().numpy().argmax(axis=1)
 y_pred = nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,1]
self.y_true = np.hstack(( self.y_true, y_true))
self.y_pred = np.hstack(( self.y_pred, y_pred))
self.score = metrics.roc_auc_score(self.y_true, self.y_pred, labels=np.array([0, 1]))
@property
def avg(self):
return self.score
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def train_loop_fn(data_loader, model, optimizer, device, scheduler=None, epoch=None):
model.train()
losses = AverageMeter()
auc = RocAucMeter()
start_time = time.time()
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids=ids,
attention_mask = mask, 
)
 loss = loss_fn(outputs, targets)
 loss.backward()
xm.optimizer_step(optimizer)
 loss = loss.detach().item()
auc.update(targets, outputs)
 losses.update(loss, ids.size(0))
if bi % 10 == 0:
xm.master_print(f'bi={bi}, loss={losses.avg:<8.4f}, auc={auc.avg:<8.4f} {time.time() -start_time:<2.2f}')
 if scheduler is not None:
scheduler.step()
del loss
del losses
del outputs
del ids
del targets
gc.collect()
model.eval()
def eval_loop_fn(data_loader, model, device):
fin_targets = []
fin_outputs = []
fin_index = []
with torch.no_grad() :
for bi, d in enumerate(data_loader):
if bi % 10 == 0:
xm.master_print(f'EVAL bi={bi}')
 ids = d["ids"]
mask = d["mask"]
targets = d["targets"]
index = d["index"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids = ids,
attention_mask = mask, 
)
 targets_np = targets.cpu().detach().numpy().argmax(axis=1 ).tolist()
outputs_np = outputs.cpu().detach().numpy() [:,1].tolist()
fin_targets.extend(targets_np)
 fin_outputs.extend(outputs_np)
 fin_index.extend(index.tolist())
 return fin_outputs, fin_targets, fin_index 
<compute_test_metric>
class SmoothLoss(nn.Module):
def __init__(self):
super(SmoothLoss, self ).__init__()
def forward(self, pred, target):
pred = pred.log_softmax(dim=1)
 return torch.mean(torch.sum(-target * pred, dim=1))
def loss_fn(outputs, targets):
return SmoothLoss()(outputs, targets)
 TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE = 32
EPOCHS = 1
LR = 2e-5
def _run() :
gc.collect()
xm.master_print('starting run')
 valid_sampler = torch.utils.data.distributed.DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=VALID_BATCH_SIZE,
sampler=valid_sampler,
drop_last=False,
num_workers=0 
)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 test_data_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=VALID_BATCH_SIZE,
sampler=test_sampler,
drop_last=False,
num_workers=0 
)
 gc.collect()
device = xm.xla_device()
model = mx.to(device)
 xm.master_print('done loading model')
 num_train_steps = int(len(df_train[0])/ TRAIN_BATCH_SIZE / xm.xrt_world_size())
 optimizer = AdamW([{'params': model.roberta.parameters() , 'lr': LR},
{'params': [param for name, param in model.named_parameters() if 'roberta' not in name], 'lr': 1e-3} ], lr=LR, weight_decay=0)
 scheduler = get_cosine_schedule_with_warmup(
 optimizer,
num_warmup_steps = 0,
num_training_steps = num_train_steps * EPOCHS 
)
 xm.master_print(f'num_train_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 for epoch in range(EPOCHS):
train_dataset = BERTDataset(df_train[epoch])
 train_sampler = torch.utils.data.distributed.DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=0,
shuffle=False 
)
 train_sampler.set_epoch(epoch)
 para_loader = pl.ParallelLoader(train_data_loader, [device])
 xm.master_print('parallel loader created...training now')
 train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler, epoch=epoch)
 del para_loader
gc.collect()
if epoch == EPOCHS-1:
xm.master_print('saving model')
 xser.save(model.state_dict() , f"{PATH}model.bin", master_only=True)
 xm.master_print('model saved')
 para_loader = pl.ParallelLoader(valid_data_loader, [device])
 o, t, i = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 auc = metrics.roc_auc_score(np.array(t), o)
 gc.collect()
del para_loader
print(f'[xla:{xm.get_ordinal() }] AUC = {auc}')
 def reduce_fn(vals):
return sum(vals)/ len(vals)
 auc = xm.mesh_reduce('auc_reduce', auc, reduce_fn)
 xm.master_print(f'AUC AVG = {auc}')
 para_loader = pl.ParallelLoader(test_data_loader, [device])
 o, t, i = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 del t
gc.collect()
return o, i 
<load_pretrained>
test_preds = np.zeros(len(df_test))
for i in range(8):
test_preds[np.load(f"test_index_{i}.npy", allow_pickle=True ).reshape(-1)] = np.load(f"test_preds_{i}.npy", allow_pickle=True ).reshape(-1 )<load_from_csv>
%%time
xgbreg = xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.3, max_delta_step=0, max_depth=6,
min_child_weight=1, monotone_constraints='() ',
n_estimators=3000, n_jobs=16, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
tree_method='exact', validate_parameters=1, verbosity=None)
 xgbreg.fit(X_train_scalled, y_train )
sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 sub['toxic'] = test_preds<save_to_csv>
y_train_pred = xgbreg.predict(X_train_scalled )
sub.to_csv('submission.csv', index=False )<save_to_csv>
r2_score(y_train, y_train_pred )
sub.to_csv('submission.csv', index=False )<import_modules>
y_val_pred = xgbreg.predict(X_val_scalled )
import numpy as np
import pandas as pd
import os<define_variables>
r2_score(y_val, y_val_pred )
LANG = "ru"
DIR = f".. /input/{LANG}-changed-subs/"
WEIGHT = 1<load_from_csv>
mae = metrics.mean_absolute_error(y_val, y_val_pred)
 mse = metrics.mean_squared_error(y_val, y_val_pred)
 rmse = np.sqrt(metrics.mean_absolute_error(y_val, y_val_pred))
print("Mean Absolute Error")
 print(mae)
 print()
print("Mean Squared Error")
 print(mse)
 print()
print("Root Mean Squared Error")
 print(rmse )
submission = pd.read_csv(".. /input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv")
 test = pd.read_csv(".. /input/jigsaw-multilingual-toxic-comment-classification/test.csv")
 sub_best = pd.read_csv(os.path.join(DIR, "sub-LB-9549.csv"))<sort_values>
evaluation = evaluation_df('Extreme Gradient Boosting with Tuning 2', mae, mse, rmse, evaluation )
files_sub = os.listdir(DIR)
 files_sub = sorted(files_sub)
 print(len(files_sub))
files_sub<load_from_csv>
df_store.isnull().sum() * 100 / df_store.shape[0]
for file in files_sub:
test[file.replace(".csv", "")] = pd.read_csv(os.path.join(DIR, file)) ["toxic"]<filter>
mapping(['Promo2'] )
test = test.loc[test["lang"]==LANG].reset_index(drop=True)
 test.head(1 )<feature_engineering>
df_train['CompetitionDistanceLog10'] = np.log10(df_train['CompetitionDistance'])
 df_train.head()
test["diff_good1"] = test[f"{LANG}-9397"] - test[f"{LANG}-9373"]
test["diff_good2"] = test[f"{LANG}-9476"] - test[f"{LANG}-9475"]
test["diff_good3"] = test[f"{LANG}-9529"] - test[f"{LANG}-9510"]
test["diff_good4"] = test[f"{LANG}-9544"] - test[f"{LANG}-9543"]
test["diff_bad1"] = test[f"{LANG}-9545"] - test[f"{LANG}-9543-from-9545"]<feature_engineering>
df_train.drop('CompetitionDistance', inplace = True, axis = 1)
 df_train.head()
test["sub_best"] = test["sub-LB-9549"]
col_comment = ["id", "content", "sub_best"]
col_diff = [column for column in test.columns if "diff" in column]
test_diff = test[col_comment + col_diff].reset_index(drop=True)
 test_diff["diff_avg"] = test_diff[col_diff].mean(axis=1 )<feature_engineering>
X = df_train.drop(['Sales', 'Store'], axis = 1)
 y = df_train['Sales']
test_diff["sub_new"] = test_diff.apply(lambda x:(1+WEIGHT*x["diff_avg"])*x["sub_best"] if x["diff_avg"]<0 else(1-WEIGHT*x["diff_avg"])*x["sub_best"] + WEIGHT*x["diff_avg"] , axis=1 )<save_to_csv>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size= 0.1, random_state = 53)
 X_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size = 0.5, random_state = 53 )
submission["toxic"] = sub_best["toxic"]
submission.loc[test["id"], "toxic"] = test_diff["sub_new"].values
submission.to_csv("submission.csv", index=False )<save_to_csv>
%%time
xgbreg = xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.3, max_delta_step=0, max_depth=6,
min_child_weight=1, monotone_constraints='() ',
n_estimators=3000, n_jobs=16, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
tree_method='exact', validate_parameters=1, verbosity=None)
 xgbreg.fit(X_train, y_train )
sub = pd.read_csv('/kaggle/input/ensemble-files/submission.csv')
 sub.to_csv('submission.csv', index=False )<import_modules>
y_train_pred = xgbreg.predict(X_train )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<categorify>
r2_score(y_train, y_train_pred )
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
tokenizer = AutoTokenizer.from_pretrained(MODEL)
 def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'])
 def build_model_PT(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC() ])
 return model
def build_model_mix(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_lang_tags = Input(shape=(4,), dtype=tf.float32, name="input_lang_tags")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = Concatenate()([cls_token, input_lang_tags])
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_lang_tags], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC() ])
 return model
def lang_embed(lang, tran):
lang_codes = {'en':'000', 'es':'100', 'fr':'010',
'it':'001', 'pt':'110', 'ru':'101',
'tr':'011'}
tran_codes = {'orig':'0', 'tran':'1'}
vec = lang_codes[lang]+tran_codes[tran]
vec = [int(v)for v in vec]
return vec
def text_process(text):
ws = text.split(' ')
 if(len(ws)>160):
text = ' '.join(ws[:160])+ ' ' + ' '.join(ws[-32:])
 return text<load_from_csv>
y_val_pred = xgbreg.predict(X_val )
test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')
 test['content'] = test['content'].apply(lambda x: text_process(x))
x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 lang_tag_test = np.array([lang_embed(row['lang'], 'orig')for _, row in test.iterrows() ])
<load_pretrained>
r2_score(y_val, y_val_pred )
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model_PT(transformer_layer, max_len=MAX_LEN)
<save_to_csv>
mae = metrics.mean_absolute_error(y_val, y_val_pred)
 mse = metrics.mean_squared_error(y_val, y_val_pred)
 rmse = np.sqrt(metrics.mean_absolute_error(y_val, y_val_pred))
print("Mean Absolute Error")
 print(mae)
 print()
print("Mean Squared Error")
 print(mse)
 print()
print("Root Mean Squared Error")
 print(rmse )
model.load_weights('/kaggle/input/mymodels/mg2mp4.h5')
 test['p1'] = model.predict(x_test, verbose=1)
<save_to_csv>
evaluation = evaluation_df('Extreme Gradient Boosting with Change in Data', mae, mse, rmse, evaluation )
model.load_weights('/kaggle/input/mymodels/en2mp1.h5')
 test['p2'] = model.predict(x_test, verbose=1)
<save_to_csv>
X = df_train.drop(['Sales', 'Year'], axis = 1)
 y = df_train['Sales']
model.load_weights('/kaggle/input/mymodels/en2mp4.h5')
 test['p3'] = model.predict(x_test, verbose=1)
<set_options>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size= 0.1, random_state = 53)
 X_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size = 0.5, random_state = 53 )
del model
K.clear_session()
gc.collect()<load_pretrained>
%%time
xgbreg = xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.3, max_delta_step=0, max_depth=5,
min_child_weight=1, monotone_constraints='() ',
n_estimators=4500, n_jobs=16, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
tree_method='exact', validate_parameters=1, verbosity=None)
 xgbreg.fit(X_train, y_train )
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model_mix(transformer_layer, max_len=MAX_LEN)
<save_to_csv>
y_train_pred = xgbreg.predict(X_train )
model.load_weights('/kaggle/input/mymodels/mixmoriggen.h5')
 test['p4'] = model.predict([x_test, lang_tag_test], verbose=1)
<save_to_csv>
r2_score(y_train, y_train_pred )
model.load_weights('/kaggle/input/mymodels/mixmorigp3.h5')
 test['p5'] = model.predict([x_test, lang_tag_test], verbose=1)
<train_on_grid>
y_val_pred = xgbreg.predict(X_val )
mybest = pd.read_csv('/kaggle/input/mybest/sub9523.csv')
 X = np.array([test.p1.values, test.p2.values, test.p3.values, test.p4.values, test.p5.values] ).T
y = mybest.toxic.values
X_T, X_val, y_T, y_val = train_test_split(X, y, test_size=0.2)
 model = LinearRegression()
model.fit(X, y)
 prds = model.predict(X_val)
 score1 = roc_auc_score(y_val.round().astype(int), prds)
 score2 = roc_auc_score(prds.round().astype(int), y_val)
 print('Validation:', score1, score2)
 prds = model.predict(X)
 score1 = roc_auc_score(y.round().astype(int), prds)
 score2 = roc_auc_score(prds.round().astype(int), y)
 print('Again my best:', score1, score2)
 test['prd'] = prds<categorify>
r2_score(y_val, y_val_pred )
p = [1.3,0.6,0.8,0.5,0.6,0.6]
out = []
for _, row in test.iterrows() :
item = [row['id'], row['prd'], row['lang']]
if(item[2]=='es'):
if(item[1]<0.7):
item[1] *= p[0]
elif(item[2]=='fr'):
if(item[1]<0.7):
item[1] *= p[1]
elif(item[2]=='ru'):
if(item[1]<0.7):
item[1] *= p[2]
elif(item[2]=='it'):
if(item[1]<0.7):
item[1] *= p[3]
elif(item[2]=='tr'):
if(item[1]<0.7):
item[1] *= p[4]
elif(item[2]=='pt'):
if(item[1]<0.7):
item[1] *= p[5]
out.append([item[0], item[1]])
 of = pd.DataFrame(out, columns=['id', 'toxic'])
 print(of.head())
 score1 = roc_auc_score(mybest.toxic.round().astype(int), of.toxic.values)
 score2 = roc_auc_score(of.toxic.round().astype(int), mybest.toxic.values)
 print('%2.4f\t%2.4f'%(100*score1, 100*score2))<feature_engineering>
y_test_pred = xgbreg.predict(X_test )
dic = {}
oft = open('/kaggle/input/profanity/Profanity.txt', "r", encoding='utf8')
 for l in oft:
ele = l.strip().lower().split(':')
 dic[ele[0]] = ele[1]
oft.close()
les, lit, ltr, lfr, lru, lpt = 1.2, 1.1, 1.3, 1.2, 1.2, 1.3
of['content'] = test['content']
of['tran'] = test['translated']
of['lang'] = test['lang']
out = []
enpros = dic['en'].split(',')
 for _, row in of.iterrows() :
if(row['lang']=='es'):
lmd = les
elif(row['lang']=='it'):
lmd = lit
elif(row['lang']=='tr'):
lmd = ltr
elif(row['lang']=='fr'):
lmd = lfr
elif(row['lang']=='ru'):
lmd = lru
else:
lmd = lpt
item = [row['id'], row['toxic']]
if(item[1]<0.5):
for w in enpros:
if(str(row['tran'] ).lower().find(w)>=0):
item[1] *= 1.2
break
ws = dic[row['lang']].split(',')
 for w in ws:
if(str(row['content'] ).lower().find(w)>=0):
item[1] *= lmd
break
out.append(item)
 of = pd.DataFrame(out, columns=['id', 'toxic'])
 score1 = roc_auc_score(mybest.toxic.round().astype(int), of.toxic.values)
 score2 = roc_auc_score(of.toxic.round().astype(int), mybest.toxic.values)
 print('%2.4f\t%2.4f'%(100*score1, 100*score2))<save_to_csv>
r2_score(y_test, y_test_pred )
of['toxic'] = mybest.toxic.values*0.8 + of.toxic.values*0.2
score1 = roc_auc_score(mybest.toxic.round().astype(int), of.toxic.values)
 score2 = roc_auc_score(of.toxic.round().astype(int), mybest.toxic.values)
 print('%2.4f\t%2.4f'%(100*score1, 100*score2))
print(of.head())
 of.to_csv('submission.csv', index=False )<install_modules>
mae = metrics.mean_absolute_error(y_val, y_val_pred)
 mse = metrics.mean_squared_error(y_val, y_val_pred)
 rmse = np.sqrt(metrics.mean_absolute_error(y_val, y_val_pred))
print("Mean Absolute Error")
 print(mae)
 print()
print("Mean Squared Error")
 print(mse)
 print()
print("Root Mean Squared Error")
 print(rmse )
!git clone https://github.com/henrique/jigsaw20.git
sys.path.append('/kaggle/working/jigsaw20' )<init_hyperparams>
evaluation = evaluation_df('Extreme Gradient Boosting with Change in Data and tuning', mae, mse, rmse, evaluation )
params = dict(
 pooling='first',
optimizer='LAMB',
batch_size=27,
lr=0.000277952,
mom_min=0.806579,
mom_max=0.922184,
div_factor=55.477,
final_div_factor=1123.49,
weight_decay=7.72285e-06,
dropout=0.4,
loss_fn='bce',
label_smoothing=0.0483175,
warm_up=1.2361,
epochs=41,
steps_per_epoch=250,
dataset='.. /input/jigsaw20-ds-tt6-36/jigsaw20_ds1789117tt6_fold5.npz',
path=f'jigsaw',
tpu_id=None, gcs=None,
seed=1083, 
)
<install_modules>
df_test = pd.read_csv('.. /input/rossmann-store-sales/test.csv')
 df_test.head()
!rm -r jigsaw20<load_from_csv>
df_test.isnull().sum()
pd.read_csv('params0.965669.csv' ).T<save_to_csv>
df_test[df_test['Open'].isnull() ]
sub = pd.read_csv('submission.csv')
 test = pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub.loc[test["lang"] == "es", "toxic"] *= 1.06
sub.loc[test["lang"] == "fr", "toxic"] *= 1.04
sub.loc[test["lang"] == "it", "toxic"] *= 0.97
sub.loc[test["lang"] == "pt", "toxic"] *= 0.96
sub.loc[test["lang"] == "tr", "toxic"] *= 0.98
sub.toxic -= sub.toxic.min()
sub.toxic /= sub.toxic.max()
sub.toxic.hist(bins=100, log=True, alpha=0.6)
 sub.to_csv('submission0.csv', index=False )<save_to_csv>
df_test[df_test['Open'].isnull() ]['Date'].value_counts()
ensemble = pd.read_csv('.. /input/jigsaw20xiwuhanjmtc2ndplacesolution/submission.csv')
 ensemble.toxic -= ensemble.toxic.min()
ensemble.toxic /= ensemble.toxic.max()
ensemble.toxic.hist(bins=100, log=True, alpha=0.6)
 sub.toxic.hist(bins=100, log=True, alpha=0.6)
 ensemble.toxic = ensemble.toxic * 0.8 + sub.toxic * 0.2
ensemble.to_csv('submission.csv', index=False )<load_from_csv>
df_test[df_test['Store'] == 622]['Open'].value_counts()
data = {k.split('/')[-1][:-4]:k for k in glob.glob('/kaggle/input/**/**.csv')}
train = pd.read_csv(data['jigsaw-toxic-comment-train'], usecols=['id', 'comment_text', 'toxic'])
 val = pd.read_csv(data['validation'], usecols=['comment_text', 'toxic'])
 test = pd.read_csv(data['test'], usecols=['id', 'content'])
 test.columns = ['id', 'comment_text']
test['toxic'] = 0.5
sub2 = pd.read_csv('.. /input/finalsubmission/submission-.9480.csv')
 sub4 = pd.read_csv('.. /input/finalsubmission/submission-.9481.csv' )<feature_engineering>
df_test['Open'] = df_test['Open'].fillna(1)
 df_test.isnull().sum()
%%time
def f_experience(c, s):
it = {'memory':10,
'influence':0.5,
'inference':0.5,
'interest':0.9,
'sentiment':1e-10,
'harmony':0.5}
exp = {}
for i in range(len(c)) :
words = set([w for w in str(c[i] ).lower().split(' ')])
 for w in words:
try:
exp[w]['influence'] = exp[w]['influence'][1:] + [s[i]]
exp[w]['inference'] += 1
exp[w]['interest'] = exp[w]['interest'][1:] + [(exp[w]['interest'][it['memory']-1] +(s[i] * it['interest'])) /2]
exp[w]['sentiment'] += s[i]
except:
m = [0.for m_ in range(it['memory'])]
exp[w] = {}
exp[w]['influence'] = m[1:] + [s[i]]
exp[w]['inference'] = 1
exp[w]['interest'] = m[1:] + [s[i] * it['interest'] / 2]
exp[w]['sentiment'] = s[i]
for w in exp:
exp[w]['sentiment'] /= exp[w]['inference'] + it['sentiment']
exp[w]['inference'] /= len(c)* it['inference']
return exp
exp = f_experience(train['comment_text'].values, train['toxic'].values )<feature_engineering>
def mapping(features):
for feature in features:
temp_dict = {}
temp_dict = pd.Series(df_store[feature].values, index = df_store['Store'] ).to_dict()
df_test[feature] = df_test['Store'].map(temp_dict )
%%time
def features(df):
df['len'] = df['comment_text'].map(len)
 df['wlen'] = df['comment_text'].map(lambda x: len(str(x ).split(' ')))
 df['influence_sum'] = df['comment_text'].map(lambda x: np.sum([np.mean(exp[w]['influence'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['influence_mean'] = df['comment_text'].map(lambda x: np.mean([np.mean(exp[w]['influence'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['inference_sum'] = df['comment_text'].map(lambda x: np.sum([exp[w]['inference'] if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['inference_mean'] = df['comment_text'].map(lambda x: np.mean([exp[w]['inference'] if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['interest_sum'] = df['comment_text'].map(lambda x: np.sum([np.mean(exp[w]['interest'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['interest_mean'] = df['comment_text'].map(lambda x: np.mean([np.mean(exp[w]['interest'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['sentiment_sum'] = df['comment_text'].map(lambda x: np.sum([exp[w]['sentiment'] if w in exp else 0.5 for w in str(x ).lower().split(' ')]))
df['sentiment_mean'] = df['comment_text'].map(lambda x: np.mean([exp[w]['sentiment'] if w in exp else 0.5 for w in str(x ).lower().split(' ')]))
return df
val = features(val)
 test= features(test )<train_on_grid>
mapping(['StoreType', 'Assortment', 'Promo2', 'CompetitionDistance'] )
col = [c for c in val if c not in ['id', 'comment_text', 'toxic']]
x1, x2, y1, y2 = model_selection.train_test_split(val[col], val['toxic'], test_size=0.3, random_state=20)
 model = ensemble.ExtraTreesClassifier(n_estimators=1000, max_depth=7, n_jobs=-1, random_state=20)
 model.fit(x1, y1)
 print(metrics.roc_auc_score(y2, model.predict_proba(x2)[:,1].clip(0.,1.)))
model.fit(val[col], val['toxic'])
 test['toxic'] = model.predict_proba(test[col])[:,1].clip(0.,1.)
sub1 = test[['id', 'toxic']]<merge>
df_test['CompetitionDistanceLog10'] = np.log10(df_test['CompetitionDistance'])
 df_test.head()
sub1.rename(columns={'toxic':'toxic1'}, inplace=True)
 sub2.rename(columns={'toxic':'toxic2'}, inplace=True)
 sub4.rename(columns={'toxic':'toxic4'}, inplace=True)
 sub3 = sub1.merge(sub2,on='id' ).merge(sub4,on='id' )<save_to_csv>
df_test.drop('CompetitionDistance', inplace = True, axis = 1)
 df_test.head()
sub3['toxic'] =(sub3['toxic1'] * 0.1)+(sub3['toxic4'] * 0.9)
 sub3['toxic'] =(sub3['toxic2'] * 0.49)+(sub3['toxic'] * 0.51)
 sub3[['id', 'toxic']].to_csv('submission.csv', index=False )<prepare_output>
df_test['DayOfYear'] = df_test['Date'].map(lambda x: datetime.datetime.strptime(str(x),'%Y-%m-%d' ).timetuple().tm_yday)
 df_test.head(10 )
test = pd.DataFrame(['Howling with Wolf on Lügenpresse'], columns=['comment_text'])
 test['id'] = test.index
test= features(test)
 test['toxic'] = model.predict_proba(test[col])[:,1].clip(0.,1.)
test[['id', 'comment_text', 'toxic']].head()<import_modules>
df_test['Date'] = pd.to_datetime(df_test['Date'], format = '%Y-%m-%d' )
import numpy as np
import pandas as pd
import xgboost as xgb
import matplotlib.pyplot as plt<load_from_csv>
df_test['Year'] = df_test['Date'].map(lambda x: x.year )
df_train = pd.read_csv(".. /input/train.csv", parse_dates=['timestamp'])
 df_test = pd.read_csv(".. /input/test.csv", parse_dates=['timestamp'])
 df_macro = pd.read_csv(".. /input/macro.csv", parse_dates=['timestamp'])
 df_train.head()<prepare_x_and_y>
df_test.drop('Date', inplace = True, axis = 1 )
y_train = df_train['price_doc'].values
id_test = df_test['id']
df_train.drop(['id', 'price_doc'], axis=1, inplace=True)
 df_test.drop(['id'], axis=1, inplace=True)
 num_train = len(df_train)
 df_all = pd.concat([df_train, df_test])
 df_all = df_all.join(df_macro, on='timestamp', rsuffix='_macro')
 print(df_all.shape)
 month_year =(df_all.timestamp.dt.month + df_all.timestamp.dt.year * 100)
 month_year_cnt_map = month_year.value_counts().to_dict()
df_all['month_year_cnt'] = month_year.map(month_year_cnt_map)
 week_year =(df_all.timestamp.dt.weekofyear + df_all.timestamp.dt.year * 100)
 week_year_cnt_map = week_year.value_counts().to_dict()
df_all['week_year_cnt'] = week_year.map(week_year_cnt_map)
 df_all['month'] = df_all.timestamp.dt.month
df_all['dow'] = df_all.timestamp.dt.dayofweek
df_all['rel_floor'] = df_all['floor'] / df_all['max_floor'].astype(float)
 df_all['rel_kitch_sq'] = df_all['kitch_sq'] / df_all['full_sq'].astype(float)
 df_all.drop(['timestamp', 'timestamp_macro'], axis=1, inplace=True )<categorify>
df_test['StateHoliday'].value_counts()
factorize = lambda t: pd.factorize(t[1])[0]
df_obj = df_all.select_dtypes(include=['object'])
 X_all = np.c_[
df_all.select_dtypes(exclude=['object'] ).values,
np.array(list(map(factorize, df_obj.iteritems())) ).T
]
print(X_all.shape)
 X_train = X_all[:num_train]
X_test = X_all[num_train:]<concatenate>
df_test['StateHoliday'] = df_test['StateHoliday'].apply(lambda x: 1 if x == 'a' else(2 if x == 'b' else(3 if x == 'c' else x)))
 df_test['StateHoliday'].value_counts()
df_numeric = df_all.select_dtypes(exclude=['object'])
 df_obj = df_all.select_dtypes(include=['object'] ).copy()
for c in df_obj:
df_obj[c] = pd.factorize(df_obj[c])[0]
df_values = pd.concat([df_numeric, df_obj], axis=1 )<prepare_x_and_y>
df_test['StateHoliday'] = df_test['StateHoliday'].map(lambda x: 0 if x == '0' else x)
 type(df_test['StateHoliday'][0] )
xgb_params = {
'eta': 0.05,
'max_depth': 8,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
dtrain = xgb.DMatrix(X_train, y_train, feature_names=df_columns)
 dtest = xgb.DMatrix(X_test, feature_names=df_columns )<train_model>
df_test_open = df_test[df_test['Open'] == 1]
df_test_closed = df_test[df_test['Open'] == 0]
model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round=num_boost_round )<save_to_csv>
df_test_closed['Sales'] = 0
y_pred = model.predict(dtest)
 df_sub = pd.DataFrame({'id': id_test, 'price_doc': y_pred})
 df_sub.to_csv('sub.csv', index=False )<import_modules>
df_test_open.drop('Open', inplace = True, axis = 1)
 df_test_closed.drop('Open', inplace = True, axis = 1 )
import numpy as np
import pandas as pd
import xgboost as xgb
import matplotlib.pyplot as plt<load_from_csv>
df_test_open.drop('Year', inplace = True, axis = 1)
 df_test_closed.drop('Year', inplace = True, axis = 1 )
df_train = pd.read_csv(".. /input/train.csv", parse_dates=['timestamp'])
 df_test = pd.read_csv(".. /input/test.csv", parse_dates=['timestamp'])
 df_macro = pd.read_csv(".. /input/macro.csv", parse_dates=['timestamp'])
 df_train.head()<prepare_x_and_y>
X = df_test_open.drop(['Id'], axis = 1 )
df_train.loc[df_train['full_sq'].isnull() ,'full_sq'] = 54
tr_full_sq = df_train['full_sq'].values
df_train['full_sq'] = np.apply_along_axis(lambda x:(x+54)/2,0,tr_full_sq)
 df_test.loc[df_test['full_sq'].isnull() ,'full_sq'] = 54
te_full_sq = df_test['full_sq'].values
df_test['full_sq'] = np.apply_along_axis(lambda x:(x+54)/2,0,te_full_sq)
 df_train['price_per_m2'] = df_train['price_doc']/df_train['full_sq'].astype(float)
 y_train = df_train['price_per_m2'].values
id_test = df_test['id']
df_train.drop(['id', 'price_doc', 'price_per_m2'], axis=1, inplace=True)
 df_test.drop(['id'], axis=1, inplace=True)
 num_train = len(df_train)
 df_all = pd.concat([df_train, df_test])
 df_all = pd.merge_ordered(df_all, df_macro, on='timestamp', how='left')
 print(df_all.shape)
 month_year =(df_all.timestamp.dt.month + df_all.timestamp.dt.year * 100)
 month_year_cnt_map = month_year.value_counts().to_dict()
df_all['month_year_cnt'] = month_year.map(month_year_cnt_map)
 week_year =(df_all.timestamp.dt.weekofyear + df_all.timestamp.dt.year * 100)
 week_year_cnt_map = week_year.value_counts().to_dict()
df_all['week_year_cnt'] = week_year.map(week_year_cnt_map)
 df_all['month'] = df_all.timestamp.dt.month
df_all['dow'] = df_all.timestamp.dt.dayofweek
df_all['rel_floor'] = df_all['floor'] / df_all['max_floor'].astype(float)
 df_all['rel_kitch_sq'] = df_all['kitch_sq'] / df_all['full_sq'].astype(float)
 df_all.drop(['timestamp'], axis=1, inplace=True )<concatenate>
cols = ['Store', 'DayOfWeek', 'Promo', 'StateHoliday', 'SchoolHoliday', 'StoreType', 'Assortment', 'DayOfYear', 'Promo2', 'CompetitionDistanceLog10']
X = X[cols]
df_numeric = df_all.select_dtypes(exclude=['object'])
 df_obj = df_all.select_dtypes(include=['object'] ).copy()
for c in df_obj:
df_obj[c] = pd.factorize(df_obj[c])[0]
df_values = pd.concat([df_numeric, df_obj], axis=1 )<prepare_x_and_y>
X_pred = xgbreg.predict(X )
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
dtrain = xgb.DMatrix(X_train, y_train, feature_names=df_columns)
 dtest = xgb.DMatrix(X_test, feature_names=df_columns )<train_model>
df_test_open['Sales'] = X_pred
model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round=num_boost_round )<save_to_csv>
df_test_final = pd.concat([df_test_open, df_test_closed] )
y_pred = model.predict(dtest)
 y_pred = y_pred*df_test['full_sq']
df_sub = pd.DataFrame({'id': id_test, 'price_doc': y_pred})
 df_sub.to_csv('sub.csv', index=False )<set_options>
sample_sub = pd.read_csv('.. /input/rossmann-store-sales/sample_submission.csv')
 sample_sub.head()
%load_ext autoreload
%autoreload 2
np.random.seed(1)
 w.filterwarnings('ignore' )<load_from_csv>
temp_dict = {}
temp_dict = pd.Series(df_test_final['Sales'].values, index = df_test_final['Id'] ).to_dict()
sample_sub['Sales'] = sample_sub['Id'].map(temp_dict )
train_file = pd.read_csv(os.path.join(main_path, "train.csv"))
test_file = pd.read_csv(os.path.join(main_path, "test.csv"))<train_model>
sample_sub.to_csv('sample_submission.csv', header = ['Id', 'Sales'], index = False )