Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<categorify>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from nltk.corpus import stopwords
from nltk.util import ngrams
from nltk.stem import WordNetLemmatizer
import re
from textblob import TextBlob
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, precision_score, recall_score, confusion_matrix
from sklearn.tree import DecisionTreeClassifier
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
import tensorflow as tf
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from gensim.models import Word2Vec
from gensim.models.keyedvectors import KeyedVectors
import time
from keras.layers import Dense, Input, Flatten, Dropout
from keras.layers import Conv1D, MaxPooling1D, Embedding
from keras.models import Sequential
from keras import losses
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
class DaeAdder(BaseEstimator, TransformerMixin):
def __init__(self,filename):
self.filename=filename
def fit(self,X,y=None):
return self
def fit_transform(self,X,y=None):
Dae_features = pd.read_feather(self.filename ).set_index('sig_id')
 return X.merge(Dae_features,how='left', on='sig_id')
 def transform(self,X):
Dae_features = pd.read_feather(self.filename ).set_index('sig_id')
 return X.merge(Dae_features,how='left', on='sig_id')
<define_variables>
train_data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 submit_data = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv' )<choose_model_class>
train_data[train_data['text'].isna() ]
Dae0_2 =DaeAdder(filename='features_0.2_altogether.fth')
<feature_engineering>
train_data.groupby('target' ).count()
res = Dae0_2.fit_transform(train )<train_model>
%matplotlib inline
class SupressControls(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
X = X[X['cp_type']!=0]
X = X.drop('cp_type', axis=1)
 return X
def transform(self, X):
X = X[X['cp_type']!=0]
X = X.drop('cp_type', axis=1)
 return X
class CatIntMapper(BaseEstimator, TransformerMixin):
def __init__(self ,col,dicti):
self.col = col
self.dicti = dicti
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 def transform(self, X):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 class NamedOutTWrapper(BaseEstimator, TransformerMixin):
def __init__(self,transformer,columns,inplace=False,prefix='_'):
self.transformer = transformer
self.cols = columns
self.inplace = inplace
self.prefix = prefix
self.transformer_name = self._get_transformer_name()
def fit(self, X, y = None):
self.transformer = self.transformer.fit(X[self.cols] , y)
 return self
def fit_transform(self, X, y = None):
transformed_columns = self.transformer.fit_transform(X[self.cols] , y)
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols].copy()
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def transform(self, X):
transformed_columns = self.transformer.transform(X[self.cols])
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols].copy()
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def _get_transformer_name(self):
return str(self.transformer.__class__ ).split('.')[-1][0:-2]
class IdentityTransformer:

 def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X
def transform(self, X):
return X
class SuppressControls(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X.loc[X['cp_type']=='trt_cp'].drop('cp_type', axis=1)
 def transform(self, X):
return X.loc[X['cp_type']=='trt_cp'].drop('cp_type', axis=1)
 class ColumnDropper(BaseEstimator, TransformerMixin):
def __init__(self, cols):
self.cols=cols
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
return X.drop(self.cols,axis=1)
 class VarianceFilter(BaseEstimator, TransformerMixin):
def __init__(self,threshold):
self.threshold = threshold
def fit(self,X,y=None):
mask = X.var() <= self.threshold
self.drop_cols = set([ col for val,col in zip(mask,X.columns)if val])
 self.drop_cols.discard('cp_type')
 return self
def transform(self,X):
return X.drop(self.drop_cols,axis=1)
 def apply_pipe_together(pipeline,train,test):
data = pd.concat([train,test])
 data = pipeline.fit_transform(data)
 train = data.loc[data.index.intersection(train.index)]
test = data.loc[data.index.intersection(test.index)]
return pipeline,train,test<categorify>
data = pd.concat([train_data, submit_data])
 data.shape
map_controls = CatIntMapper('cp_type',{'ctl_vehicle': 0, 'trt_cp': 1})
 map_dose = CatIntMapper('cp_dose',{'D1': 1, 'D2': 0})
 map_time = CatIntMapper('cp_time',{24: 0, 48: 1, 72: 2})
<normalization>
data['text'] = data['text'].apply(lambda x: re.sub(re.compile(r'https?\S+'), '', x))
data['text'] = data['text'].apply(lambda x: re.sub(re.compile(r'[\//:,.!?@&\-'\`"\_
\
data['text'] = data['text'].apply(lambda x: re.sub(re.compile(r'<.*?>'), '', x))
data['text'] = data['text'].apply(lambda x: re.sub(re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE), '', x))
data['text'] = data['text'].apply(lambda x: re.sub(re.compile(r'\d'), '', x))
data['text'] = data['text'].apply(lambda x: re.sub(re.compile(r'[^\w]'), ' ', x))
data['text'] = data['text'].str.lower()
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<feature_engineering>
clean_train = data[0:train_data.shape[0]]
clean_submit = data[train_data.shape[0]:-1]
X_train, X_test, y_train, y_test = train_test_split(clean_train['text'], clean_train['target'],
test_size = 0.2, random_state = 4 )
def groupkfold_indexer(train,target_columns,n_splits=10,random_state=12347,**kwargs):
folds = train.copy()
groups =pd.read_csv('.. /input/inferreddrugexperimentclustermoa/aggclus4_results.csv',index_col='sig_id')['aggclus1-clusters']
gkf = GroupKFold(n_splits=n_splits)
 folds[ 'kfold']=0
for f,(t_idx, v_idx)in enumerate(gkf.split(X=train.sample(len(train),random_state=random_state), y=train[target_columns].sample(len(train),random_state=random_state),groups= groups.loc[train.index].sample(len(train),random_state=random_state))):
folds.iloc[v_idx,-1] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 return folds<choose_model_class>
def tfidf(words):
tfidf_vectorizer = TfidfVectorizer()
data_feature = tfidf_vectorizer.fit_transform(words)
 return data_feature, tfidf_vectorizer
X_train_tfidf, tfidf_vectorizer = tfidf(X_train.tolist())
 X_test_tfidf = tfidf_vectorizer.transform(X_test.tolist() )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size1=388,hidden_size2=512,drop_rate1=0.0,drop_rate2=0.3,drop_rate3=0.3):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(drop_rate1)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size1))
self.batch_norm2 = nn.BatchNorm1d(hidden_size1)
 self.dropout2 = nn.Dropout(drop_rate2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size1, hidden_size2))
self.batch_norm3 = nn.BatchNorm1d(hidden_size2)
 self.dropout3 = nn.Dropout(drop_rate3)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size2, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<train_model>
lr_tfidf = LogisticRegression(class_weight = 'balanced', solver = 'lbfgs', n_jobs = -1)
 lr_tfidf.fit(X_train_tfidf, y_train)
 y_predicted_lr = lr_tfidf.predict(X_test_tfidf )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
if not scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, scheduler, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 if scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step(final_loss)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<train_model>
def score_metrics(y_test, y_predicted):
accuracy = accuracy_score(y_test, y_predicted)
 precision = precision_score(y_test, y_predicted)
 recall = recall_score(y_test, y_predicted)
 print("accuracy = %0.3f, precision = %0.3f, recall = %0.3f" %(accuracy, precision, recall))
def run_training(X_train,y_train,X_valid,y_valid,X_test,fold, seed,verbose=False,**kwargs):
seed_everything(seed)
 train_dataset = MoADataset(X_train, y_train)
 valid_dataset = MoADataset(X_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features= X_train.shape[1] ,
num_targets= y_train.shape[1],hidden_size1=hidden_size1,hidden_size2=hidden_size2,
**kwargs 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,patience=3)
 loss_val = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 if verbose:
print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model,scheduler, loss_val, validloader, DEVICE)
 if verbose:
print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof = valid_preds
elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
torch.save(model.state_dict() , f"{exp_name}_SEED{seed}_FOLD{fold}.pth")
 testdataset = TestDataset(X_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions 
<init_hyperparams>
score_metrics(y_test, y_predicted_lr )
DEVICE = torch.device('cuda:0')
 EPOCHS = 70
BATCH_SIZE = 1024
LEARNING_RATE = 6e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
hidden_size1=2048
hidden_size2=2048<define_variables>
pipeline = Pipeline([ 
('clf', DecisionTreeClassifier(splitter='random', class_weight='balanced'))
])
 parameters = {
'clf__max_depth':(150,160,165),
'clf__min_samples_split':(18,20,23),
'clf__min_samples_leaf':(5,6,7)
 }
df_tfidf = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=-1, scoring='f1')
 df_tfidf.fit(X_train_tfidf, y_train)
 print(df_tfidf.best_estimator_.get_params() )
transformers_list=[map_dose,map_time,Dae0_2,SuppressControls() ]<create_dataframe>
y_predicted_dt = df_tfidf.predict(X_test_tfidf )
def run_k_fold(folds,target_cols,test,transformers_list,NFOLDS, seed,verbose=False,**kwargs):
train = folds
test_ = test
oof = train[target_cols].copy()
oof = oof*0
predictions = pd.DataFrame(0,columns=target_cols,index=test.index)
 for fold in range(NFOLDS):
train_df = train[train['kfold'] != fold]
valid_df = train[train['kfold'] == fold]
feature_cols = [col for col in train_df.columns if not(col in target_cols+['kfold'])]
pipeline_val = make_pipeline(*transformers_list)
 X_train, y_train = train_df[feature_cols], train_df[target_cols]
X_valid, y_valid = valid_df[feature_cols], valid_df[target_cols].values
X_train = pipeline_val.fit_transform(X_train,y_train)
 feature_cols = [col for col in X_train.columns if not(col in target_cols+['kfold'])]
X_train = X_train.values
X_valid = pipeline_val.transform(X_valid)
 valid_index = X_valid.index
X_valid = X_valid.values
y_train = y_train.values
X_test = pipeline_val.transform(test_)
 test_index = X_test.index
X_test = X_test[feature_cols].values
oof_, pred_ = run_training(X_train,y_train,X_valid,y_valid,X_test,fold, seed,verbose,**kwargs)
 oof.loc[valid_index] = oof_
predictions.loc[test_index] += pred_ / NFOLDS
return oof, predictions<find_best_params>
score_metrics(y_test, y_predicted_dt )
params={}<load_from_csv>
!pip install gensim -i http://pypi.douban.com/simple --trusted-host pypi.douban.com
SEED = [0]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 pipeline_test = make_pipeline(*transformers_list)
 pipeline_test,train , test = apply_pipe_together(pipeline_test,train,test)
 transformers_list=[IdentityTransformer() ]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
folds = groupkfold_indexer(train,target_cols,n_splits=NFOLDS)
 oof_, predictions_ = run_k_fold(folds,target_cols,test,transformers_list,NFOLDS, seed,verbose=True,**params)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 test[target_cols] = predictions 
<compute_test_metric>
stop_words = stopwords.words('english')
 for word in ['us','no','yet']:
stop_words.append(word)
 data_list = []
text_series = data['text']
for i in range(len(text_series)) :
content = text_series.iloc[i]
cutwords = [word for word in content.split(' ')if word not in stop_words if len(word)!= 0]
data_list.append(cutwords )
y_true = train[target_cols].values
y_pred = oof
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred.iloc[:, i],labels=[0,1])
 score +=(score_ / len(target_cols))
print("CV log_loss: ", score)
<feature_engineering>
for i in range(len(data_list)) :
content = data_list[i]
if len(content)<1:
print(i )
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 y_true = train[target_cols].values
y_pred = train[target_cols].copy()
y_pred[target_cols] = 0
y_pred.loc[oof.index] = oof
y_pred.loc[train.cp_type=='ctl_vehicle'] = 0
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred.iloc[:, i],labels=[0,1])
 score +=(score_ / len(target_cols))
print("CV log_loss: ", score)
<save_to_csv>
word2vec_path='./GoogleNews-vectors-negative300.bin.gz'
word2vec_model = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True )
sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission.set_index('sig_id',inplace=True)
 test_features.set_index('sig_id',inplace=True)
 test_features = test_features.loc[sample_submission.index]
sub = sample_submission.drop(columns=target_cols ).merge(test[target_cols], on='sig_id', how='left' ).fillna(0)
 sub.loc[test_features['cp_type']=='ctl_vehicle', target_cols] =0
sub.to_csv('./submission.csv', index=True )<set_options>
def get_textVector(data_list, word2vec, textsVectors_list):
for i in range(len(data_list)) :
words_perText = data_list[i]
if len(words_perText)< 1:
words_vector = [np.zeros(300)]
else:
words_vector = [word2vec.wv[k] if k in word2vec_model else np.zeros(300)for k in words_perText]
text_vector = np.array(words_vector ).mean(axis=0)
 textsVectors_list.append(text_vector)
 return textsVectors_list
sys.path.append('.. /input/iterativestratification')
 warnings.filterwarnings('ignore')
 os.listdir('.. /input/lish-moa')
 pd.set_option('max_columns', 2000 )<define_variables>
textsVectors_list = []
get_textVector(data_list, word2vec_model, textsVectors_list)
 X = np.array(textsVectors_list )
n_comp_GENES = 463
n_comp_CELLS = 60
VarianceThreshold_for_FS = 0.9
Dropout_Model = 0.25
QuantileTransformer_n_quantiles = 500
print('n_comp_GENES', n_comp_GENES, 'n_comp_CELLS', n_comp_CELLS, 'total', n_comp_GENES + n_comp_CELLS )<load_from_csv>
pd.isnull(X ).any()
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
word2vec_X = X[0:train_data.shape[0]]
y = data['target'][0:train_data.shape[0]]
word2vec_submit = X[train_data.shape[0]:-1]
X_train_word2vec, X_test_word2vec, y_train_word2vec, y_test_word2vec = train_test_split(word2vec_X, y,
test_size = 0.2, random_state = 4 )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<categorify>
word2vec_lr = LogisticRegression(class_weight = 'balanced', solver = 'lbfgs', n_jobs = -1)
 word2vec_lr.fit(X_train_word2vec, y_train_word2vec)
 y_predicted_word2vec_lr = word2vec_lr.predict(X_test_word2vec )
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=QuantileTransformer_n_quantiles,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
score_metrics(y_test_word2vec, y_predicted_word2vec_lr )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<create_dataframe>
compare_list = []
for(i,j)in zip(y_test_word2vec, y_predicted_word2vec_lr):
k = i - j
compare_list.append(k)
 wrong_num = [i for i,j in enumerate(compare_list)if j != 0]
text_series[0:train_data.shape[0]][wrong_num]
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<concatenate>
lenlen = []
for i in range(len(data_list)) :
content = data_list[i]
perlen = len(content)
 lenlen.append(perlen)
 print(max(lenlen))
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<concatenate>
max_sequence_length = 26
embedding_dim = 300
data = train_features.append(test_features)
 data<create_dataframe>
tokenizer = Tokenizer()
tokenizer.fit_on_texts(data_list)
 sequences = tokenizer.texts_to_sequences(data_list)
 word_index = tokenizer.word_index
cnn_data = pad_sequences(sequences, maxlen = max_sequence_length)
 cnn_label = to_categorical(np.asarray(train_data['target']))
print('len of word_index:', len(word_index))
print('shape of data tensor:', cnn_data.shape)
 print('shape of label tensoe:', cnn_label.shape )
var_thresh = VarianceThreshold(VarianceThreshold_for_FS)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape<merge>
trainCNN_data = cnn_data[0:train_data.shape[0]]
X_train_cnn, X_test_cnn, y_train_cnn, y_test_cnn = train_test_split(trainCNN_data, cnn_label,
test_size = 0.2, random_state = 4)
 X_cnn, X_val_cnn, y_cnn, y_val_cnn = train_test_split(X_train_cnn, y_train_cnn,
test_size = 0.2, random_state = 4 )
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]<drop_column>
CNNmodel = Sequential()
CNNmodel.add(Embedding(len(word_index)+1, embedding_dim, input_length = max_sequence_length))
CNNmodel.add(Conv1D(filters=250, kernel_size=3, strides=1, padding='valid', activation = 'relu'))
CNNmodel.add(MaxPooling1D(pool_size=3))
CNNmodel.add(Flatten())
 CNNmodel.add(Dense(embedding_dim, activation='relu'))
CNNmodel.add(Dropout(0.8))
CNNmodel.add(Dense(cnn_label.shape[1], activation='sigmoid'))
CNNmodel.summary()
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
CNNmodel.compile(optimizer='adam', loss=losses.binary_crossentropy, metrics=['accuracy'])
 history = CNNmodel.fit(X_cnn, y_cnn, epochs=3, validation_data=(X_val_cnn, y_val_cnn))
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<split>
test_loss, test_acc = CNNmodel.evaluate(X_test_cnn, y_test_cnn, verbose=2)
 print('test loss:',test_loss)
 print('test acc:',test_acc )
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds<categorify>
embedding_matrix = np.zeros(( len(word_index)+ 1, embedding_dim))
for word, i in word_index.items() :
if word in word2vec_model:
embedding_matrix[i] = np.asarray(word2vec_model.wv[word] )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
embedding_layer = Embedding(len(word_index)+1,
embedding_dim,
weights = [embedding_matrix],
input_length = max_sequence_length,
trainable = False )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
pass
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
model = Sequential()
model.add(embedding_layer)
 model.add(Conv1D(filters=150, kernel_size=3, strides=1, padding='valid', activation = 'relu'))
model.add(MaxPooling1D(pool_size=3))
model.add(Flatten())
 model.add(Dense(embedding_dim, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(cnn_label.shape[1], activation='sigmoid'))
model.summary()
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<categorify>
model.compile(optimizer='adam', loss=losses.binary_crossentropy, metrics=['accuracy'])
 history = model.fit(X_cnn, y_cnn, epochs=10, validation_data=(X_val_cnn, y_val_cnn))
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
test_loss, test_acc = model.evaluate(X_test_cnn, y_test_cnn, verbose=2)
 print('test loss:',test_loss)
 print('test acc:',test_acc )
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
tf.__version__
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1500<normalization>
hub.__version__
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(Dropout_Model)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(Dropout_Model)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))<prepare_x_and_y>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f".. /input/20221107fe-fs-dnn-model/FOLD{fold}_.pth"))
model.to(DEVICE)
 torch.manual_seed(seed)
 oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] = valid_preds
print(valid_loss)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 del model
torch.cuda.empty_cache()
gc.collect()
return oof, predictions<categorify>
import tensorflow as tf
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<define_variables>
def bert_encode(texts, bert_layer, max_len=128):
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len - 2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 input_ids = tokens + [0]* pad_len
all_tokens.append(input_ids)
 masks = [1]*len(input_sequence)+ [0]* pad_len
all_masks.append(masks)
 segments = [0]* max_len
all_segments.append(segments)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_model(bert_layer, max_len = 128, lr = 1e-5):
input_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,name="input_word_ids")
 input_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,name="input_mask")
 segment_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,name="segment_ids")
 pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 dense_out = Dense(1,activation="relu" )(pooled_output)
 drop_out = tf.keras.layers.Dropout(0.8 )(dense_out)
 out = Dense(1,activation="sigmoid" )(pooled_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 adam = tf.keras.optimizers.Adam(lr)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
 return model
def plot_curve(history):
plt.plot(history.history['accuracy'], label='accuracy')
 plt.plot(history.history['val_accuracy'], label='val_accuracy')
 plt.xlabel('Epoch')
 plt.ylabel('Accuracy')
 plt.ylim([0.5,1])
 plt.legend()
plt.show()
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions 
<merge>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<save_to_csv>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 train_input = bert_encode(train.text.values, bert_layer, max_len=128)
 train_labels = np.array(train.target )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<define_variables>
test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 test_input = bert_encode(test.text.values, bert_layer, max_len=128)
 model.load_weights('model.h5')
 test_pred = model.predict(test_input )
<load_from_csv><EOS> 
submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 submission['target'] = np.round(test_pred ).astype('int')
 submission.to_csv('submission.csv', index=False)
 submission.groupby('target' ).count()