Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<find_best_params>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import string
%%time
%%capture
env = janestreet.make_env()
learn.model.eval()
preds = []
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
x_tt[:, 1:] = for_loop(fillna_npwhere_njit, x_tt[:, 1:], f_mean)
 pred = 0.
 pred = learn.model(t.from_numpy(x_tt ).to(device, t.float)).sigmoid() [0][-1].item()
preds.append(pred)
 action = 1 if pred >=.5 else 0
pred_df.action = action
else:
pred_df.action = 0
env.predict(pred_df )<compute_test_metric>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 sub_sample = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
preds = np.array(preds)
 preds.mean() , preds.std() , sum(preds >=.5), sum(preds < 5)
<define_variables>
all_data_na =(train.isnull().sum() / len(train)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head()
TRAINING = False<load_from_csv>
all_data_na =(test.isnull().sum() / len(test)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head()
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('weight>0' ).reset_index(drop=True)
 train.fillna(train.median() ,inplace=True)
 train['feature_stock_id_sum'] = train['feature_41'] + train['feature_42'] + train['feature_43']
train['feature_1_2_cross'] = train['feature_1']/(train['feature_2']+1e-5)
 NUM_TRAIN_EXAMPLES = len(train )<feature_engineering>
train.duplicated().sum()
features = [c for c in train.columns if 'feature' in c]
f_mean = np.nanmean(train[features[1:]].values,axis=0 )<define_variables>
train['keyword'].value_counts()
PATH = '.. /input/densenetneu'<define_variables>
test['keyword'].value_counts()
resp_cols = [c for c in train.columns if 'resp' in c]<choose_model_class>
train['location'].value_counts()
def mish(x):
return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)) )(x)
 tf.keras.utils.get_custom_objects().update({'mish': tf.keras.layers.Activation(mish)})
 def create_model(input_shape):
inp = tf.keras.layers.Input(input_shape)
 tmp = tf.keras.layers.BatchNormalization()(inp)
 xs = [tmp]
for _ in range(10):
if len(xs)> 1:
tmp = tf.keras.layers.Concatenate(axis=-1 )(xs)
 else:
tmp = xs[0]
tmp = tf.keras.layers.Dense(64,activation='mish' )(tmp)
 tmp = tf.keras.layers.BatchNormalization()(tmp)
 tmp = tf.keras.layers.Dropout(0.2 )(tmp)
 xs.append(tmp)
 output = tf.keras.layers.Dense(len(resp_cols),activation='sigmoid' )(tf.keras.layers.Concatenate()(xs))
model = tf.keras.models.Model(inp,output)
 optimizer = tfa.optimizers.RectifiedAdam(1e-3)
 model.compile(optimizer, loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.001),
metrics=[tf.keras.metrics.BinaryAccuracy(name='binary_accuracy')])
 return model<feature_engineering>
for col in ['keyword','location']:
train[col] = train[col].fillna('None')
 test[col] = test[col].fillna('None')
 def clean_loc(x):
if x == 'None':
return 'None'
elif x == 'Earth' or x =='Worldwide' or x == 'Everywhere':
return 'World'
elif 'New York' in x or 'NYC' in x:
return 'New York'
elif 'Washington' in x and 'D' in x and 'C' in x:
return 'Washington DC'
elif 'United States' in x or 'USA' in x:
return 'USA'
elif 'United Kingdom' in x or 'UK' in x or 'Britain' in x:
return 'UK'
elif x in top_loc:
return x
else: return 'Others'
train['location_clean'] = train['location'].apply(lambda x: clean_loc(str(x)))
 test['location_clean'] = test['location'].apply(lambda x: clean_loc(str(x)) )
def set_all_seeds(seed):
np.random.seed(seed)
 random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed )<normalization>
test_str = train.loc[0, 'text']
def clean_text(text):
text = re.sub(r'https?://\S+', '', text)
 text = re.sub(r'
',' ', text)
 text = re.sub('\s+', ' ', text ).strip()
return text
print("Original text: " + test_str)
 print("Cleaned text: " + clean_text(test_str))
class NeutralizeTransform:
def __init__(self,proportion=1.0):
self.proportion = proportion
def fit(self,X,y):
self.lms = []
self.mean_exposure = np.mean(y,axis=0)
 self.y_shape = y.shape[-1]
for x in X.T:
scores = x.reshape(( -1,1))
exposures = y
exposures = np.hstack(( exposures, np.array([np.mean(scores)] * len(exposures)).reshape(-1, 1)))
 transform = np.linalg.lstsq(exposures, scores, rcond=None)[0]
self.lms.append(transform)
 def transform(self,X,y=None):
out = []
for i,transform in enumerate(self.lms):
x = X[:,i]
scores = x.reshape(( -1,1))
exposures = np.repeat(self.mean_exposure,len(x),axis=0 ).reshape(( -1,self.y_shape))
exposures = np.concatenate([exposures,np.array([np.mean(scores)] * len(exposures)).reshape(( -1,1)) ],axis=1)
 correction = self.proportion * exposures.dot(transform)
 out.append(x - correction.ravel())
 return np.asarray(out ).T
def fit_transform(self,X,y):
self.fit(X,y)
 return self.transform(X,y)
<categorify>
def find_hashtags(tweet):
return " ".join([match.group(0)[1:] for match in re.finditer(r"
def find_mentions(tweet):
return " ".join([match.group(0)[1:] for match in re.finditer(r"@\w+", tweet)])or 'no'
def find_links(tweet):
return " ".join([match.group(0)[:] for match in re.finditer(r"https?://\S+", tweet)])or 'no'
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 def remove_punctuations(text):
punctuations = '@
for p in punctuations:
text = text.replace(p, f' {p} ')
 text = text.replace('...', '...')
 if '...' not in text:
text = text.replace('.. ', '...')
 return text
def process_text(df):
df['text_clean'] = df['text'].apply(lambda x: clean_text(x))
df['hashtags'] = df['text'].apply(lambda x: find_hashtags(x))
df['mentions'] = df['text'].apply(lambda x: find_mentions(x))
df['links'] = df['text'].apply(lambda x: find_links(x))
df['text_clean'] = df['text_clean'].apply(lambda x: remove_emoji(x))
df['text_clean'] = df['text_clean'].apply(lambda x: remove_punctuations(x))
return df
train = process_text(train)
 test = process_text(test )
%%time
if TRAINING:
mask = train[features].isna()
train.fillna(0,inplace=True)
 for feature in features:
nt = NeutralizeTransform(proportion=0.25)
 train[feature] = nt.fit_transform(train[feature].values.reshape(( -1,1)) ,
train['resp'].values.reshape(( -1,1)))
 pd.to_pickle(nt,f'NeutralizeTransform_{feature}.pkl')
 train[mask] = np.nan
else:
nts = []
for feature in features:
nt = pd.read_pickle(f'{PATH}/NeutralizeTransform_{feature}.pkl')
 nts.append(nt )<set_options>
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact", "acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}
gc.collect()<train_model>
nltk.download('punkt' )
X_tr = train.query('date<440')[features].values
y_tr =(train.query('date<440')[resp_cols].values > 0 ).astype(int)
 X_val = train.query('date>460')[features].values
y_val =(train.query('date>460')[resp_cols].values > 0 ).astype(int)
 del train
gc.collect()
if TRAINING:
metric = {}
for seed in [6, 28, 496, 8128]:
set_all_seeds(seed)
 model = create_model(X_tr.shape[-1])
 hist = model.fit(X_tr,y_tr,
validation_data=(X_val,y_val),
epochs=200,
batch_size=8192,
callbacks=[tf.keras.callbacks.EarlyStopping('val_binary_accuracy',mode='max',patience=20,restore_best_weights=True),
tf.keras.callbacks.ReduceLROnPlateau('val_binary_accuracy',mode='max',patience=10,cooldown=5)])
 model.save_weights(f'model_{seed}.tf')
 metric[seed] = max(hist.history['val_binary_accuracy'])
 tf.keras.backend.clear_session()
print(metric)
 else:
models = []
for seed in [6, 28, 496, 8128]:
model = create_model(X_tr.shape[-1])
 model.load_weights(f'{PATH}/model_{seed}.tf')
 model.call = tf.function(model.call, experimental_relax_shapes=True)
 models.append(model)
<feature_engineering>
def convert_abbrev(word):
return abbreviations[word.lower() ] if word.lower() in abbreviations.keys() else word
def convert_abbrev_in_text(text):
tokens = word_tokenize(text)
 tokens = [convert_abbrev(word)for word in tokens]
text = ' '.join(tokens)
 return text
if not TRAINING:
f = np.median
janestreet.competition.make_env.__called__ = False
env = janestreet.make_env()
th = 0.50
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
test_df['feature_stock_id_sum'] = test_df['feature_41'] + test_df['feature_42'] + test_df['feature_43']
test_df['feature_1_2_cross'] = test_df['feature_1']/(test_df['feature_2']+1e-5)
 x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
for i in range(len(nts)) :
x_tt[:,i] = nts[i].transform(np.expand_dims(x_tt[:,i],0))
p = f(np.mean([model(x_tt,training=False ).numpy() for model in models],axis=0))
pred_df.action = np.where(p > th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<feature_engineering>
train['text_clean'] = train['text_clean'].apply(lambda x: convert_abbrev_in_text(x))
test['text_clean'] = test['text_clean'].apply(lambda x: convert_abbrev_in_text(x))
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('weight>0' ).reset_index(drop=True)
 train.fillna(train.median() ,inplace=True)
 train['feature_stock_id_sum'] = train['feature_41'] + train['feature_42'] + train['feature_43']
train['feature_1_2_cross'] = train['feature_1']/(train['feature_2']+1e-5)
 NUM_TRAIN_EXAMPLES = len(train )<define_variables>
contractions = {
"ain't": "am not",
"aren't": "are not",
"can't": "cannot",
"can't've": "cannot have",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"couldn't've": "could not have",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hadn't've": "had not have",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'd've": "he would have",
"he'll": "he will",
"he's": "he is",
"how'd": "how did",
"how'll": "how will",
"how's": "how is",
"i'd": "i would",
"i'll": "i will",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd": "it would",
"it'll": "it will",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"must've": "must have",
"mustn't": "must not",
"needn't": "need not",
"oughtn't": "ought not",
"shan't": "shall not",
"sha'n't": "shall not",
"she'd": "she would",
"she'll": "she will",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"that'd": "that would",
"that's": "that is",
"there'd": "there had",
"there's": "there is",
"they'd": "they would",
"they'll": "they will",
"they're": "they are",
"they've": "they have",
"wasn't": "was not",
"we'd": "we would",
"we'll": "we will",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"where'd": "where did",
"where's": "where is",
"who'll": "who will",
"who's": "who is",
"won't": "will not",
"wouldn't": "would not",
"you'd": "you would",
"you'll": "you will",
"you're": "you are",
"thx" : "thanks"
}
features = [c for c in train.columns if 'feature' in c]
f_mean = np.nanmedian(train[features[1:]].values,axis=0 )<define_variables>
def remove_contractions(text):
return contractions[text.lower() ] if text.lower() in contractions.keys() else text
TRAINING = False
PATH = '.. /input/highwayjs'<define_variables>
train['text_clean'] = train['text_clean'].apply(lambda x:remove_contractions(x))
test['text_clean'] = test['text_clean'].apply(lambda x: remove_contractions(x))
resp_cols = [c for c in train.columns if 'resp' in c]<choose_model_class>
def create_stat(df):
df['text_len'] = df['text_clean'].apply(len)
 df['word_count'] = df["text_clean"].apply(lambda x: len(str(x ).split()))
df['stop_word_count'] = df['text_clean'].apply(lambda x: len([w for w in str(x ).lower().split() if w in STOPWORDS]))
df['punctuation_count'] = df['text_clean'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
df['hashtag_count'] = df['hashtags'].apply(lambda x: len(str(x ).split()))
df['mention_count'] = df['mentions'].apply(lambda x: len(str(x ).split()))
df['link_count'] = df['links'].apply(lambda x: len(str(x ).split()))
df['caps_count'] = df['text_clean'].apply(lambda x: sum(1 for c in str(x)if c.isupper()))
df['caps_ratio'] = df['caps_count'] / df['text_len']
return df
train = create_stat(train)
 test = create_stat(test)
 print(train.shape, test.shape )
def mish(x):
return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)) )(x)
 tf.keras.utils.get_custom_objects().update({'mish': tf.keras.layers.Activation(mish)})
 def create_model(input_shape):
inp = tf.keras.layers.Input(input_shape)
 tmp = tf.keras.layers.BatchNormalization()(inp)
 xs = [tmp]
for _ in range(10):
if len(xs)> 1:
tmp = tf.keras.layers.Concatenate(axis=-1 )(xs)
 else:
tmp = xs[0]
tmp = tf.keras.layers.Dense(32,activation='mish' )(tmp)
 tmp = tf.keras.layers.BatchNormalization()(tmp)
 tmp = tf.keras.layers.Dropout(0.2 )(tmp)
 xs.append(tmp)
 output = tf.keras.layers.Dense(len(resp_cols),activation='sigmoid' )(tf.keras.layers.Concatenate()(xs))
model = tf.keras.models.Model(inp,output)
 optimizer = tfa.optimizers.RectifiedAdam(1e-3)
 model.compile(optimizer, loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.001),
metrics=[tf.keras.metrics.BinaryAccuracy(name='binary_accuracy')])
 return model<feature_engineering>
train.corr() ['target'].drop('target' ).sort_values()
def set_all_seeds(seed):
np.random.seed(seed)
 random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed )<train_model>
stopwords = set(STOPWORDS)
X_tr = train.query('date<440')[features].values
y_tr =(train.query('date<440')[resp_cols].values > 0 ).astype(int)
 X_val = train.query('date>460')[features].values
y_val =(train.query('date>460')[resp_cols].values > 0 ).astype(int)
 if TRAINING:
metric = {}
for seed in [6, 28, 496, 8128]:
set_all_seeds(seed)
 model = create_model(X_tr.shape[-1])
 hist = model.fit(X_tr,y_tr,
validation_data=(X_val,y_val),
epochs=200,
batch_size=8192,
callbacks=[tf.keras.callbacks.EarlyStopping('val_binary_accuracy',mode='max',patience=20)])
 model.save_weights(f'model_{seed}.tf')
 metric[seed] = max(hist.history['binary_accuracy'])
 print(metric)
 else:
models = []
for seed in [6, 28, 496, 8128]:
model = create_model(X_tr.shape[-1])
 model.load_weights(f'{PATH}/model_{seed}.tf')
 model.call = tf.function(model.call, experimental_relax_shapes=True)
 models.append(model)
<categorify>
nltk.download('punkt' )
blend = np.asarray([0.55722,0.55301,0.56278,0.55624])
 blend = blend/sum(blend )<feature_engineering>
pip install category_encoders
if not TRAINING:
f = np.median
janestreet.competition.make_env.__called__ = False
env = janestreet.make_env()
th = 0.503
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
test_df['feature_stock_id_sum'] = test_df['feature_41'] + test_df['feature_42'] + test_df['feature_43']
test_df['feature_1_2_cross'] = test_df['feature_1']/(test_df['feature_2']+1e-5)
 x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
p = sum(blend*np.asarray([f(model(x_tt,training=False ).numpy())for model in models]))
pred_df.action = np.where(p > th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<load_from_csv>
features = ['keyword', 'location_clean']
encoder = ce.TargetEncoder(cols=features)
 encoder.fit(train[features],train['target'])
 train = train.join(encoder.transform(train[features] ).add_suffix('_target'))
test = test.join(encoder.transform(test[features] ).add_suffix('_target'))
SEED = 1111
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
batch_size = 5000
hidden_units = [150, 150, 150]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
clf = create_mlp(
 len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate 
)
 clf.fit(X_train, y_train, epochs=300, batch_size=2000,verbose=1)
 models = []
models.append(clf)
 th = 0.5000
f = np.median
models = models[-3:]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<set_options>
vec_links = CountVectorizer(min_df = 2, analyzer = 'word', token_pattern = r'https?://\S+')
 link_vec = vec_links.fit_transform(train['links'])
 link_vec_test = vec_links.transform(test['links'])
 X_train_link = pd.DataFrame(link_vec.toarray() , columns=vec_links.get_feature_names())
 X_test_link = pd.DataFrame(link_vec_test.toarray() , columns=vec_links.get_feature_names())
 vec_men = CountVectorizer(min_df = 2)
 men_vec = vec_men.fit_transform(train['mentions'])
 men_vec_test = vec_men.transform(test['mentions'])
 X_train_men = pd.DataFrame(men_vec.toarray() , columns=vec_men.get_feature_names())
 X_test_men = pd.DataFrame(men_vec_test.toarray() , columns=vec_men.get_feature_names())
 vec_hash = CountVectorizer(min_df = 2)
 hash_vec = vec_hash.fit_transform(train['hashtags'])
 hash_vec_test = vec_hash.transform(test['hashtags'])
 X_train_hash = pd.DataFrame(hash_vec.toarray() , columns=vec_hash.get_feature_names())
 X_test_hash = pd.DataFrame(hash_vec_test.toarray() , columns=vec_hash.get_feature_names())
 print(X_train_link.shape, X_train_men.shape, X_train_hash.shape )
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
 warnings.filterwarnings('ignore' )<import_modules>
hash_rank =(X_train_hash.transpose().dot(train['target'])/ X_train_hash.sum(axis=0)).sort_values(ascending=False)
 print('Hashtags with which 100% of Tweets are disasters: ')
 print(list(hash_rank[hash_rank==1].index))
print('Total: ' + str(len(hash_rank[hash_rank==1])))
 print('Hashtags with which 0% of Tweets are disasters: ')
 print(list(hash_rank[hash_rank==0].index))
print('Total: ' + str(len(hash_rank[hash_rank==0])) )
print("torch version:", torch.__version__ )<set_options>
vec_text = TfidfVectorizer(min_df = 8, ngram_range =(1,2), stop_words='english')
 text_vec = vec_text.fit_transform(train['text_clean'])
 text_vec_test = vec_text.transform(test['text_clean'])
 X_train_text = pd.DataFrame(text_vec.toarray() , columns=vec_text.get_feature_names())
 X_test_text = pd.DataFrame(text_vec_test.toarray() , columns=vec_text.get_feature_names())
 print(X_train_text.shape )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<define_variables>
train = train.join(X_train_link, rsuffix='_link')
 train = train.join(X_train_men, rsuffix='_mention')
 train = train.join(X_train_hash, rsuffix='_hashtag')
 train = train.join(X_train_text, rsuffix='_text')
 test = test.join(X_test_link, rsuffix='_link')
 test = test.join(X_test_men, rsuffix='_mention')
 test = test.join(X_test_hash, rsuffix='_hashtag')
 test = test.join(X_test_text, rsuffix='_text')
 print(train.shape, test.shape )
ROOT = ".. /input/lish-moa"<load_from_csv>
text_clean_train=train['text_clean']
text_clean_test=test['text_clean']
df_train_features = pd.read_csv(f'{ROOT}/train_features.csv')
 df_train_targets_scored = pd.read_csv(f'{ROOT}/train_targets_scored.csv')
 df_train_targets_nonscored = pd.read_csv(f'{ROOT}/train_targets_nonscored.csv')
 df_train_drug = pd.read_csv(f'{ROOT}/train_drug.csv')
 df_test_features = pd.read_csv(f'{ROOT}/test_features.csv')
 df_sample_submission = pd.read_csv(f'{ROOT}/sample_submission.csv' )<define_variables>
features_to_drop = ['id', 'keyword','location','text','location_clean','text_clean', 'hashtags', 'mentions','links']
scaler = MinMaxScaler()
Target = train.target
train = train.drop(columns = features_to_drop + ['target'])
 test = test.drop(columns = features_to_drop)
col_genes = [col for col in df_train_features.columns.to_list() if col[:2] == "g-"]
col_cells = [col for col in df_train_features.columns.to_list() if col[:2] == "c-"]
col_cats = ["cp_time", "cp_dose"]
col_targets = df_train_targets_scored.columns.to_list() [1:]
print("-"*25)
 print("Number of Gene columns: ", len(col_genes))
print("Number of Cell columns: ", len(col_cells))
print("Number of Categorical columns:", len(col_cats))
print("-"*25)
 print("Number of Target columns: ", len(col_targets))
print("-"*25 )<count_unique_values>
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
print("Data explore:")
 print("-"*25)
 print("Number of unique in sig_id: %5d" % df_train_features.sig_id.unique().shape[0], "| Examples:", df_train_features.sig_id.unique() [:3])
 print("Number of unique in cp_type: %5d" % df_train_features.cp_type.unique().shape[0], "| Examples:", df_train_features.cp_type.unique() [:3])
 print("Number of unique in cp_time: %5d" % df_train_features.cp_time.unique().shape[0], "| Examples:", df_train_features.cp_time.unique() [:3])
 print("Number of unique in cp_dose: %5d" % df_train_features.cp_dose.unique().shape[0], "| Examples:", df_train_features.cp_dose.unique() [:3])
 print("-"*25)
 print("Most frequent targets:")
 print("-"*25)
 df_train_targets_scored.sum() [1:].sort_values(ascending=False )<merge>
X_train, val_X, Y_train, val_y = train_test_split(train, Target,test_size=0.1,random_state=1 )
df_train = df_train_features.merge(df_train_targets_scored, how="left", on="sig_id")
 df_train = df_train.merge(df_train_drug, on="sig_id", how="left")
 df_train = df_train.dropna()
df_test = df_test_features.copy()
print(df_train.shape)
 print(df_test.shape)
 df_train.head(2 )<groupby>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
NFOLDS = 5
SEED = 42
if "fold" in df_train.columns: del df_train['fold']
vc = df_train.drug_id.value_counts()
vc1 = vc.loc[vc<=18].index.sort_values()
vc2 = vc.loc[vc>18].index.sort_values()
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=SEED)
 tmp = df_train.groupby('drug_id')[col_targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[col_targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=SEED)
 tmp = df_train.loc[df_train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[col_targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 df_train['fold'] = df_train.drug_id.map(dct1)
 df_train.loc[df_train.fold.isna() ,'fold'] =\
df_train.loc[df_train.fold.isna() ,'sig_id'].map(dct2)
 df_train.fold = df_train.fold.astype('int8' )<categorify>
pip install sentencepiece
df_train.loc[:, 'cp_dose'] = df_train.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df_test.loc[:, 'cp_dose'] = df_test.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 col_input = col_cats + col_cells + col_genes<categorify>
random_state_split = 21
Dropout_num = 0
learning_rate = 6e-6
valid = 0.2
epochs_num = 3
batch_size_num = 16
target_corrected = False
target_big_corrected = False
class MoADataset:
def __init__(self, features_cat, features_gene, features_cell, targets):
self.features_cat = features_cat
self.features_gene = features_gene
self.features_cell = features_cell
self.targets = targets
def __len__(self):
return(self.features_cat.shape[0])
 def __getitem__(self, idx):
dct = {
'x1' : torch.tensor(self.features_cat[idx, :], dtype=torch.float),
'x2' : torch.tensor(self.features_gene[idx, :], dtype=torch.float),
'x3' : torch.tensor(self.features_cell[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features_cat, features_gene, features_cell):
self.features_cat = features_cat
self.features_gene = features_gene
self.features_cell = features_cell
def __len__(self):
return(self.features_cat.shape[0])
 def __getitem__(self, idx):
dct = {
'x1' : torch.tensor(self.features_cat[idx, :], dtype=torch.float),
'x2' : torch.tensor(self.features_gene[idx, :], dtype=torch.float),
'x3' : torch.tensor(self.features_cell[idx, :], dtype=torch.float),
}
return dct<train_model>
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
input1, input2, input3, targets = data['x1'].to(device), data['x2'].to(device), data['x3'].to(device), data['y'].to(device)
 outputs = model(input1, input2, input3)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
if scheduler: scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
input1, input2, input3, targets = data['x1'].to(device), data['x2'].to(device), data['x3'].to(device), data['y'].to(device)
 outputs = model(input1, input2, input3)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
input1, input2, input3 = data['x1'].to(device), data['x2'].to(device), data['x3'].to(device)
 with torch.no_grad() :
outputs = model(input1, input2, input3)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<compute_test_metric>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
def logloss(y_true, y_pred, eps=1e-15):
y_pred = np.clip(y_pred, eps, 1 - eps)
 return -(y_true * np.log(y_pred)+(1-y_true)* np.log(1-y_pred)).mean()<choose_model_class>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
if Dropout_num == 0:
out = Dense(1, activation='sigmoid' )(clf_output)
 else:
x = Dropout(Dropout_num )(clf_output)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=6e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model
class Model(nn.Module):
def __init__(self, num_features_cat, num_features_gene, num_features_cell, num_targets, hidden_size, dropout_rate=0.5, activation=F.relu):
super(Model, self ).__init__()
def get_sequential_layers(num_features, n_layers, hidden_size, dropout_rate):
layer_list = []
in_ = num_features
for i in range(n_layers):
layer_list.append(
 torch.nn.Sequential 
(
 torch.nn.BatchNorm1d(in_),
torch.nn.Dropout(dropout_rate),
torch.nn.Linear(in_, hidden_size), 
)
)
 in_ = hidden_size
return layer_list
self.layer_list_cat = torch.nn.ModuleList(get_sequential_layers(num_features_cat , 2, hidden_size, dropout_rate))
self.layer_list_gene = torch.nn.ModuleList(get_sequential_layers(num_features_gene, 4, hidden_size, dropout_rate))
self.layer_list_cell = torch.nn.ModuleList(get_sequential_layers(num_features_cell, 3, hidden_size, dropout_rate))
self.activation = activation
self.batch_norm_out = nn.BatchNorm1d(hidden_size*3)
 self.dropout_out = nn.Dropout(dropout_rate)
 self.dense_out = nn.Linear(hidden_size*3, num_targets)
 def forward(self, x1, x2, x3):
for batchnorm, dropout, linear in self.layer_list_cat:
x1 = batchnorm(x1)
 x1 = dropout(x1)
 x1 = linear(x1)
 x1 = self.activation(x1)
 for batchnorm, dropout, linear in self.layer_list_gene:
x2 = batchnorm(x2)
 x2 = dropout(x2)
 x2 = linear(x2)
 x2 = self.activation(x2)
 for batchnorm, dropout, linear in self.layer_list_cell:
x3 = batchnorm(x3)
 x3 = dropout(x3)
 x3 = linear(x3)
 x3 = self.activation(x3)
 x = torch.cat([x1, x2, x3], dim=-1)
 x = self.batch_norm_out(x)
 x = self.dropout_out(x)
 x = self.dense_out(x)
 return x<prepare_x_and_y>
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
def run_training(fold, seed, PARAMS, df_train, DEVICE):
seed_everything(seed)
 train_ = df_train.copy()
trn_idx = train_[train_['fold'] != fold].index
val_idx = train_[train_['fold'] == fold].index
train_df = train_[train_['fold'] != fold].reset_index(drop=True)
 valid_df = train_[train_['fold'] == fold].reset_index(drop=True)
 x_train_cat, x_train_gene, x_train_cell, y_train = train_df[col_cats].values, train_df[col_genes].values, train_df[col_cells].values, train_df[col_targets].values
x_valid_cat, x_valid_gene, x_valid_cell, y_valid = valid_df[col_cats].values, valid_df[col_genes].values, valid_df[col_cells].values, valid_df[col_targets].values
train_dataset = MoADataset(x_train_cat, x_train_gene, x_train_cell, y_train)
 valid_dataset = MoADataset(x_valid_cat, x_valid_gene, x_valid_cell, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=PARAMS["BATCH_SIZE"], shuffle=True, drop_last=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=PARAMS["BATCH_SIZE"], shuffle=False)
 model = Model(
 num_features_cat=len(col_cats),
num_features_gene=len(col_genes),
num_features_cell=len(col_cells),
num_targets=len(col_targets),
hidden_size=PARAMS["hidden_size"],
dropout_rate=PARAMS["dropout_rate"],
activation=PARAMS["activation"], 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=PARAMS["LEARNING_RATE"], weight_decay=PARAMS["WEIGHT_DECAY"])
 scheduler = None
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 if VERBOSE and epoch % VERBOSE == 0:
print(f"FOLD: %d, EPOCH: %3d | train_loss: %.5f | valid_loss: %.5f"%(fold,epoch,train_loss,valid_loss))
if valid_loss < best_loss:
if epoch > 1:
os.rename(f"SEED_{seed}_FOLD{fold}_0.pth", f"SEED_{seed}_FOLD{fold}_1.pth")
 best_loss = valid_loss
torch.save(model.state_dict() , f"SEED_{seed}_FOLD{fold}_0.pth")
 early_step = 0
elif(EARLY_STOP == True):
early_step += 1
if(early_step >= EARLY_STOPPING_STEPS):
break
x_test_cat, x_test_gene, x_test_cell = df_test[col_cats].values, df_test[col_genes].values, df_test[col_cells].values
testdataset = TestDataset(x_test_cat, x_test_gene, x_test_cell)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=PARAMS["BATCH_SIZE"], shuffle=False)
 oof = np.zeros(( df_train.shape[0], len(col_targets)))
 predictions = np.zeros(( df_test.shape[0], len(col_targets)))
 for nb in range(NBEST):
model = Model(
 num_features_cat=len(col_cats),
num_features_gene=len(col_genes),
num_features_cell=len(col_cells),
num_targets=len(col_targets),
hidden_size=PARAMS["hidden_size"],
dropout_rate=PARAMS["dropout_rate"],
activation=PARAMS["activation"], 
)
 model_path = f"SEED_{seed}_FOLD{fold}_{nb}.pth"
if not os.path.isfile(model_path):
continue
model.load_state_dict(torch.load(model_path))
model.to(DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] += valid_preds / NBEST
predictions += inference_fn(model, testloader, DEVICE)/ NBEST
score = logloss(y_valid, oof[val_idx])
 print("FOLD: %d, Log Loss: %.5f"%(fold, score))
print("-"*15)
 return oof, predictions, score 
<train_model>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
def run_k_fold(NFOLDS, seed, PARAMS, df_train, DEVICE):
oof = np.zeros(( df_train.shape[0], len(col_targets)))
 predictions = np.zeros(( df_test.shape[0], len(col_targets)))
 scores = []
for fold in range(NFOLDS):
torch.cuda.empty_cache()
oof_, pred_, score_ = run_training(fold, seed, PARAMS, df_train, DEVICE)
 predictions += pred_ / NFOLDS
oof += oof_
scores.append(score_)
 return oof, predictions, scores 
<init_hyperparams>
train_input = bert_encode(text_clean_train, tokenizer, max_len=160)
 test_input = bert_encode(text_clean_test, tokenizer, max_len=160)
 train_labels =Target.values
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 101
EARLY_STOPPING_STEPS = 30
EARLY_STOP = True
NFOLDS = 5
VERBOSE = 10
NBEST = 2
SEED = 42
PARAMS = {
'BATCH_SIZE': 256,
'LEARNING_RATE': 1e-2,
'WEIGHT_DECAY': 1e-6,
'hidden_size': 512,
'dropout_rate': 0.4,
'activation': F.relu,
}<feature_engineering>
checkpoint = ModelCheckpoint('model_BERT.h5', monitor='val_loss', save_best_only=True)
 train_history = model_BERT.fit(
 train_input, train_labels,
validation_split = 0.2,
epochs = 3,
callbacks=[checkpoint],
batch_size = 16
)
oof = np.zeros(( df_train.shape[0], len(col_targets)))
 predictions = np.zeros(( df_test.shape[0], len(col_targets)))
 st = time()
oof, predictions, scores_ = run_k_fold(NFOLDS, SEED, PARAMS, df_train, DEVICE)
 score_oof = logloss(df_train[col_targets].values, oof)
 print("-"*15)
 print("STD: %.6f | OOF: %6f | CV: %6f | done in %.1f min"%(np.std(scores_), np.mean(scores_), score_oof,(time() - st)/60))
df_test[col_targets] = predictions<save_to_csv>
model_BERT.load_weights('model_BERT.h5')
 test_pred_BERT = model_BERT.predict(test_input)
 test_pred_BERT_int = test_pred_BERT.round().astype('int' )
df_test.loc[df_test.cp_type == "ctl_vehicle", col_targets] = 0
df_test[["sig_id"] + col_targets].to_csv("submission.csv", index=False )<import_modules>
train_pred_BERT = model_BERT.predict(train_input)
 train_pred_BERT_int = train_pred_BERT.round().astype('int' )
import os
import random
import numpy as np
import pandas as pd
import tensorflow as tf
from sklearn.metrics import log_loss
from sklearn.model_selection import KFold<define_variables>
pred = pd.DataFrame(test_pred_BERT, columns=['preds'])
 pred.plot.hist()
SEED = 9
def all_seeds(s):
random.seed(s)
 os.environ['PYTHONHASHSEED'] = str(s)
 np.random.seed(s)
 tf.random.set_seed(s)
 all_seeds(SEED )<load_from_csv>
sub_sample['target'] = test_pred_BERT_int
test_df = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_df = pd.read_csv('.. /input/lish-moa/train_features.csv')
 tr_target_df = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 sample_df = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 target_cols = tr_target_df.columns[1:]
N_TARGETS = len(target_cols )<prepare_x_and_y>
sub_sample.to_csv("submission.csv", index=False, header=True )