Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<import_modules>
tweets = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
import numpy as np
import os
import shutil
import matplotlib.pyplot as plt
import zipfile
import tensorflow as tf
import xml.etree.ElementTree as ET
from tqdm import tqdm
from keras.models import *
from keras.layers.core import *
from keras.layers import *
from keras.layers.advanced_activations import LeakyReLU
from keras.initializers import RandomNormal
from keras.optimizers import Adam
from keras import backend as K
from PIL import Image<init_hyperparams>
tweets[tweets.sent_len > 8]
tweets[tweets.word_len > 50]
NOISE_SIZE = 64
LR_D = 0.0001
LR_G = 0.0002
BATCH_SIZE = 64
EPOCHS = 650
BETA1 = 0.5
SEED = 4250
random_dim = 128
img_rows = 64
img_cols = 64
channels = 3
SEED = 4250
np.random.seed(SEED)
 random_dim = 128<define_variables>
stop = set(stopwords.words('english'))
corpus0 = []
[corpus0.append(word.lower())for tweet in tweets[tweets.target == 0].text for word in word_tokenize(tweet)]
corpus1 = []
[corpus1.append(word.lower())for tweet in tweets[tweets.target == 1].text for word in word_tokenize(tweet)]
def count_top_stopwords(corpus):
stopwords_freq = {}
for word in corpus:
if word in stop:
if word in stopwords_freq:
stopwords_freq[word] += 1
else:
stopwords_freq[word] = 1
topwords = sorted(stopwords_freq.items() , key=lambda item: item[1], reverse=True)[:10]
x,y = zip(*topwords)
 return x,y
x0,y0 = count_top_stopwords(corpus0)
 x1,y1 = count_top_stopwords(corpus1)
 plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
 plt.bar(x0,y0, color=color[0])
 plt.title('Top Stopwords for Disaster Tweets')
 plt.subplot(1,2,2)
 plt.bar(x1,y1, color=color[1])
 plt.title('Top Stopwords for Non-Disaster Tweets')
ROOT_DIR = '.. /input/'
IMAGES_DIR = ROOT_DIR + 'all-dogs/all-dogs/'
BREEDS_DIR = ROOT_DIR + 'annotation/Annotation/'
IMAGES = os.listdir(IMAGES_DIR)
 BREEDS = os.listdir(BREEDS_DIR)
 print('Total Images: {}'.format(len(IMAGES)))
 print('Total Annotations: {}'.format(len(BREEDS)) )<load_pretrained>
stop = ENGLISH_STOP_WORDS.union(stop)
 def remove_url(txt):
return " ".join(re.sub("([^0-9A-Za-z \t])|(\w+:\/\/\S+)", "", txt ).split())
 corpus0 = []
[corpus0.append(word.lower())for tweet in tweets[tweets.target == 0].text for word in word_tokenize(remove_url(tweet)) ]
corpus0 = list(filter(lambda x: x not in stop, corpus0))
corpus1 = []
[corpus1.append(word.lower())for tweet in tweets[tweets.target == 1].text for word in word_tokenize(remove_url(tweet)) ]
corpus1 = list(filter(lambda x: x not in stop, corpus1))
a = Counter(corpus0 ).most_common()
df0 = pd.DataFrame(a, columns=['Word','Count'])
 a = Counter(corpus1 ).most_common()
df1 = pd.DataFrame(a, columns=['Word','Count'])
 plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
 sns.barplot(x='Word',y='Count',data=df0.head(10), color=color[1] ).set_title('Most Common Words for Non-Disasters')
 plt.xticks(rotation=45)
 plt.subplot(1,2,2)
 sns.barplot(x='Word',y='Count',data=df1.head(10), color=color[0] ).set_title('Most Common Words for Disasters')
 plt.xticks(rotation=45)
def load_images() :
all_images = np.zeros(( 22250, 64, 64, 3))
index = 0
for breed in BREEDS:
for dog in os.listdir(BREEDS_DIR + breed):
try: img = Image.open(IMAGES_DIR + dog + '.jpg')
 except: continue
tree = ET.parse(BREEDS_DIR + breed + '/' + dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 xdelta = xmax - xmin
ydelta = ymax - ymin
if xdelta >= 64 and ydelta >= 64:
img2 = img.crop(( xmin, ymin, xmax, ymax))
img2 = img2.resize(( 64, 64), Image.ANTIALIAS)
 image = np.asarray(img2)
 all_images[index,:] =(image.astype(np.float32)- 127.5)/127.5
index += 1
if index % 1000 == 0:
print('Processed Images: {}'.format(index))
print('Total Processed Images: {}'.format(index))
return all_images<find_best_params>
tweets['polarity'] = [TextBlob(tweet ).sentiment.polarity for tweet in tweets.text]
tweets['subjectivity'] = [TextBlob(tweet ).sentiment.subjectivity for tweet in tweets.text]
tweets['exclaimation_num'] = [tweet.count('!')for tweet in tweets.text]
tweets['questionmark_num'] = [tweet.count('?')for tweet in tweets.text]
def count_url_hashtag_mention(text):
urls_num = len(re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', text))
word_tokens = text.split()
hash_num = len([word for word in word_tokens if word[0] == '
mention_num = len([word for word in word_tokens if word[0] == '@' and word.count('@')== 1])
 return urls_num, hash_num, mention_num
url_num, hash_num, mention_num = zip(*[count_url_hashtag_mention(tweet)for tweet in tweets.text])
 tweets = tweets.assign(url_num = url_num, hash_num = hash_num, mention_num = mention_num)
 contractions = ["'t", "'re", "'s", "'d", "'ll", "'ve", "'m"]
tweets['contraction_num'] = [sum([tweet.count(cont)for cont in contractions])for tweet in tweets.text]
class AdamWithWeightnorm(Adam):
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr *=(1./(1.+ self.decay * K.cast(self.iterations, K.floatx())))
 t = K.cast(self.iterations + 1, K.floatx())
 lr_t = lr * K.sqrt(1.- K.pow(self.beta_2, t)) /(1.- K.pow(self.beta_1, t))
shapes = [K.get_variable_shape(p)for p in params]
ms = [K.zeros(shape)for shape in shapes]
vs = [K.zeros(shape)for shape in shapes]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
ps = K.get_variable_shape(p)
 if len(ps)>1:
V, V_norm, V_scaler, g_param, grad_g, grad_V = get_weightnorm_params_and_grads(p, g)
 V_scaler_shape = K.get_variable_shape(V_scaler)
 m_g = K.zeros(V_scaler_shape)
 v_g = K.zeros(V_scaler_shape)
 m_g_t =(self.beta_1 * m_g)+(1.- self.beta_1)* grad_g
v_g_t =(self.beta_2 * v_g)+(1.- self.beta_2)* K.square(grad_g)
 new_g_param = g_param - lr_t * m_g_t /(K.sqrt(v_g_t)+ self.epsilon)
 self.updates.append(K.update(m_g, m_g_t))
self.updates.append(K.update(v_g, v_g_t))
m_t =(self.beta_1 * m)+(1.- self.beta_1)* grad_V
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(grad_V)
 new_V_param = V - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
if getattr(p, 'constraint', None)is not None:
new_V_param = p.constraint(new_V_param)
 add_weightnorm_param_updates(self.updates, new_V_param, new_g_param, p, V_scaler)
 else:
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_weightnorm_params_and_grads(p, g):
ps = K.get_variable_shape(p)
 V_scaler_shape =(ps[-1],)
 V_scaler = K.ones(V_scaler_shape)
 norm_axes = [i for i in range(len(ps)- 1)]
V = p / tf.reshape(V_scaler, [1] * len(norm_axes)+ [-1])
 V_norm = tf.sqrt(tf.reduce_sum(tf.square(V), norm_axes))
g_param = V_scaler * V_norm
grad_g = tf.reduce_sum(g * V, norm_axes)/ V_norm
grad_V = tf.reshape(V_scaler, [1] * len(norm_axes)+ [-1])* \ 
(g - tf.reshape(grad_g / V_norm, [1] * len(norm_axes)+ [-1])* V)
 return V, V_norm, V_scaler, g_param, grad_g, grad_V
def add_weightnorm_param_updates(updates, new_V_param, new_g_param, W, V_scaler):
ps = K.get_variable_shape(new_V_param)
 norm_axes = [i for i in range(len(ps)- 1)]
new_V_norm = tf.sqrt(tf.reduce_sum(tf.square(new_V_param), norm_axes))
new_V_scaler = new_g_param / new_V_norm
new_W = tf.reshape(new_V_scaler, [1] * len(norm_axes)+ [-1])* new_V_param
updates.append(K.update(W, new_W))
updates.append(K.update(V_scaler, new_V_scaler))
def data_based_init(model, input):
if type(input)is dict:
feed_dict = input
elif type(input)is list:
feed_dict = {tf_inp: np_inp for tf_inp,np_inp in zip(model.inputs,input)}
else:
feed_dict = {model.inputs[0]: input}
if model.uses_learning_phase and K.learning_phase() not in feed_dict:
feed_dict.update({K.learning_phase() : 1})
 layer_output_weight_bias = []
for l in model.layers:
trainable_weights = l.trainable_weights
if len(trainable_weights)== 2:
W,b = trainable_weights
assert(l.built)
 layer_output_weight_bias.append(( l.name,l.get_output_at(0),W,b))
sess = K.get_session()
for l,o,W,b in layer_output_weight_bias:
print('Performing data dependent initialization for layer ' + l)
 m,v = tf.nn.moments(o, [i for i in range(len(o.get_shape())-1)])
 s = tf.sqrt(v + 1e-10)
 updates = tf.group(W.assign(W/tf.reshape(s,[1]*(len(W.get_shape())-1)+[-1])) , b.assign(( b-m)/s))
sess.run(updates, feed_dict )<choose_model_class>
tweets.keyword.fillna('None', inplace=True)
 def decontraction(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
tweets.text = [decontraction(tweet)for tweet in tweets.text]
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 print(remove_emoji("OMG there is a volcano eruption!!! 😭😱😷"))
tweets.text = tweets.text.apply(lambda x: remove_emoji(x))
def create_generator_model() :
init = RandomNormal(mean = 0.0, stddev = 0.02)
 model = Sequential()
start_shape = NOISE_SIZE * 4 * 4
model.add(Dense(start_shape, kernel_initializer = init, input_dim = random_dim))
model.add(Reshape(( 4, 4, 64)))
 model.add(UpSampling2D())
 model.add(Conv2D(128, kernel_size = 3, padding = "same", kernel_initializer = init))
model.add(ReLU())
 model.add(UpSampling2D())
 model.add(Conv2D(128, kernel_size = 3, padding = "same", kernel_initializer = init))
model.add(ReLU())
 model.add(UpSampling2D())
 model.add(Conv2D(128, kernel_size = 3, padding = "same", kernel_initializer = init))
model.add(ReLU())
 model.add(UpSampling2D())
 model.add(Conv2D(128, kernel_size = 3, padding = "same", kernel_initializer = init))
model.add(ReLU())
 model.add(Conv2D(3, kernel_size = 3, activation = 'tanh', padding = 'same', kernel_initializer=init))
model.compile(loss = 'binary_crossentropy', optimizer = AdamWithWeightnorm(lr = LR_G, beta_1 = BETA1))
print(model.summary())
 return model<choose_model_class>
tweets.text = tweets.text.apply(lambda x: remove_url(x))
def remove_punct(text):
new_punct = re.sub('\ |\!|\?', '', punctuation)
 table=str.maketrans('','',new_punct)
 return text.translate(table)
 tweets.text = tweets.text.apply(lambda x: remove_punct(x))
def replace_amp(text):
text = re.sub(r" amp ", " and ", text)
 return text
tweets.text = tweets.text.apply(lambda x: replace_amp(x))
def create_discriminator_model() :
input_shape =(img_rows, img_cols, channels)
 init = RandomNormal(mean = 0.0, stddev = 0.02)
 model = Sequential()
model.add(Conv2D(128, kernel_size = 3, strides = 2, padding = 'same', kernel_initializer = init, input_shape = input_shape))
model.add(LeakyReLU(0.2))
model.add(Dropout(0.25))
model.add(Conv2D(128, kernel_size = 3, strides = 2, padding = 'same', kernel_initializer = init))
model.add(LeakyReLU(0.2))
model.add(Dropout(0.25))
model.add(Conv2D(128, kernel_size = 3, strides = 2, padding = 'same', kernel_initializer = init))
model.add(LeakyReLU(0.2))
model.add(Dropout(0.25))
model.add(Conv2D(128, kernel_size = 3, strides = 2, padding = 'same', kernel_initializer = init))
model.add(LeakyReLU(0.2))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(1, activation = 'sigmoid', kernel_initializer = init))
model.compile(loss = 'binary_crossentropy', optimizer = AdamWithWeightnorm(lr = LR_D , beta_1 = BETA1))
print(model.summary())
 return model<choose_model_class>
tweets = pd.read_csv('.. /input/twitter-logo/tweets_segmented.csv' )
def create_gan_model(discriminator, random_dim, generator):
discriminator.trainable = False
gan_input = Input(shape =(random_dim,))
generator_output = generator(gan_input)
 gan_output = discriminator(generator_output)
 gan_model = Model(inputs = gan_input, outputs = gan_output)
 gan_model.compile(loss = 'binary_crossentropy', optimizer = AdamWithWeightnorm(lr = LR_G, beta_1 = BETA1))
print(gan_model.summary())
 return gan_model<define_variables>
lemmatizer = WordNetLemmatizer()
def lemma(text):
words = word_tokenize(text)
 return ' '.join([lemmatizer.lemmatize(w.lower() , pos='v')for w in words])
 tweets.text = tweets.text.apply(lambda x: lemma(x))
def generator_input(latent_dim, n_samples):
input = np.random.randn(latent_dim * n_samples)
 input = input.reshape(( n_samples, latent_dim))
return input<choose_model_class>
def generate_ngrams(text, n):
words = word_tokenize(text)
 return [' '.join(ngram)for ngram in list(get_data(ngrams(words, n)))if not all(w in stop for w in ngram)]
def get_data(gen):
try:
for elem in gen:
yield elem
except(RuntimeError, StopIteration):
return
def train_model(epochs = 1, batch_size = 128):
x_train = load_images()
batch_count = x_train.shape[0] / batch_size
generator = create_generator_model()
discriminator = create_discriminator_model()
gan_model = create_gan_model(discriminator, random_dim, generator)
 discriminator_fake_hist, discriminator_real_hist, generator_hist = [], [], []
for e in range(epochs):
script_stopper_counter = 0
print('======================== Epoch {} ============================='.format(e))
for _ in tqdm(range(int(batch_count))):
discriminator_fake_loss, discriminator_real_loss = [], []
for _ in range(2):
X_fake = generator.predict(generator_input(random_dim, batch_size))
y_fake = np.zeros(batch_size)
 y_fake[:] = 0
discriminator.trainable = True
d_fake_loss = discriminator.train_on_batch(X_fake, y_fake)
 X_real = x_train[np.random.randint(0, x_train.shape[0], size = batch_size)]
y_real = np.zeros(batch_size)
 y_real[:] = 0.9
discriminator.trainable = True
d_real_loss = discriminator.train_on_batch(X_real, y_real)
 discriminator_fake_loss.append(d_fake_loss)
 discriminator_real_loss.append(d_real_loss)
 noise = generator_input(random_dim, batch_size)
 y_gen = np.ones(batch_size)
 discriminator.trainable = False
generator_loss = gan_model.train_on_batch(noise, y_gen)
 d_fake_loss = %.4f, d_real_loss = %.4f g_loss = %.4f' % \
discriminator_fake_hist.append(np.mean(discriminator_fake_loss))
discriminator_real_hist.append(np.mean(discriminator_real_loss))
generator_hist.append(generator_loss)
 if np.mean(discriminator_fake_loss)> 10:
script_stopper_counter += 1
if e % 100 == 0:
plot_generated_images(e, generator)
 if script_stopper_counter > 160:
plot_generated_images(e, generator)
 break
plot_loss(discriminator_fake_hist, discriminator_real_hist, generator_hist)
 z = zipfile.PyZipFile('images.zip', mode = 'w')
 for k in range(10000):
generated_images = generator.predict(np.random.normal(0, 1, size = [1, random_dim]))
image = Image.fromarray(((generated_images + 1)* 127.5 ).astype('uint8' ).reshape(64, 64, 3))
f = str(k)+'.png'
image.save(f, 'PNG')
 z.write(f)
 os.remove(f)
 if k % 1000 == 0:
print(k)
 z.close()<train_model>
bigrams_disaster = tweets[tweets.target==1].text.apply(lambda x: generate_ngrams(x, 2))
bigrams_ndisaster = tweets[tweets.target==0].text.apply(lambda x: generate_ngrams(x, 2))
bigrams_d_dict = {}
for bgs in bigrams_disaster:
for bg in bgs:
if bg in bigrams_d_dict:
bigrams_d_dict[bg] += 1
else:
bigrams_d_dict[bg] = 1
bigrams_d_df = pd.DataFrame(bigrams_d_dict.items() , columns=['Bigrams','Count'])
 bigrams_nd_dict = {}
for bgs in bigrams_ndisaster:
for bg in bgs:
if bg in bigrams_nd_dict:
bigrams_nd_dict[bg] += 1
else:
bigrams_nd_dict[bg] = 1
bigrams_nd_df = pd.DataFrame(bigrams_nd_dict.items() , columns=['Bigrams','Count'] )
train_model(EPOCHS, BATCH_SIZE )<define_variables>
trigrams_disaster = tweets[tweets.target==1].text.apply(lambda x: generate_ngrams(x, 3))
trigrams_ndisaster = tweets[tweets.target==0].text.apply(lambda x: generate_ngrams(x, 3))
trigrams_d_dict = {}
for tgs in trigrams_disaster:
for tg in tgs:
if tg in trigrams_d_dict:
trigrams_d_dict[tg] += 1
else:
trigrams_d_dict[tg] = 1
trigrams_d_df = pd.DataFrame(trigrams_d_dict.items() , columns=['Trigrams','Count'])
 trigrams_nd_dict = {}
for tgs in trigrams_ndisaster:
for tg in tgs:
if tg in trigrams_nd_dict:
trigrams_nd_dict[tg] += 1
else:
trigrams_nd_dict[tg] = 1
trigrams_nd_df = pd.DataFrame(trigrams_nd_dict.items() , columns=['Trigrams','Count'] )
ComputeLB = False
DogsOnly = True
ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
imagesIn2 = np.zeros(( 25000,64,64))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img3 = img2
img3 = img3.transpose(Image.FLIP_LEFT_RIGHT)
 img3 = img3.resize(( 64,64), Image.ANTIALIAS)
 img3 = img3.convert('LA')
 img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 imagesIn2[idxIn,:,:] = np.asarray(img3)[:,:,0]
namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
imagesIn2 = imagesIn2[idx,:,:]
namesIn = np.array(namesIn)[idx]
else:
IMAGES = np.sort(IMAGES)
 np.random.seed(810)
 x = np.random.choice(np.arange(20579),10000)
 np.random.seed(None)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1 
<data_type_conversions>
def remove_stopwords(text):
word_tokens = word_tokenize(text)
 return ' '.join([w.lower() for w in word_tokens if not w.lower() in stop])
 tweets['text_nostopwords'] = tweets.text.apply(lambda x: remove_stopwords(x))
plt.imshow(Image.fromarray(( imagesIn[101] ).astype('uint8' ).reshape(( 64,64,3))))<import_modules>
mask = np.array(Image.open('.. /input/twitter-logo/Twitter-Logo_white.png'))
reverse = mask[...,::-1,:]
def wc_words(target, mask=mask):
words = [word.lower() for tweet in tweets[tweets.target == target].text_nostopwords for word in tweet.split() ]
words = list(filter(lambda w: w != 'like', words))
words = list(filter(lambda w: w != 'new', words))
words = list(filter(lambda w: w != 'people', words))
dict = {}
for w in words:
if w in dict:
dict[w] += 1
else:
dict[w] = 1
my_cloud = WordCloud(background_color='white', stopwords=stop, mask=mask, random_state=0 ).generate_from_frequencies(dict)
 plt.subplot(1,2,target+1)
 plt.imshow(my_cloud, interpolation='bilinear')
 plt.axis("off")
 plt.figure(figsize=(15,10))
wc_words(0)
 plt.title('Non-Disaster')
 wc_words(1, reverse)
 plt.title('Disaster')
 plt.show()
from keras.models import Model
from keras.layers import Input, Dense, Conv2D, Reshape, Flatten, concatenate
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from keras.optimizers import SGD, Adam 
<choose_model_class>
pd.reset_option('max_colwidth')
 tweets.drop('text_nostopwords', axis=1, inplace=True)
 tweets.head()
def define_discriminator(in_shape=(28,28,1)) :
model = Sequential()
model.add(Conv2D(128,(3,3), strides=(2,2), padding='same', input_shape=in_shape))
model.add(LeakyReLU(alpha=0.2))
model.add(Conv2D(128,(3,3), strides=(2,2), padding='same'))
model.add(LeakyReLU(alpha=0.2))
model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(1, activation='sigmoid'))
opt = Adam(lr=0.0002, beta_1=0.5)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
 return model<categorify>
X_train, X_val, y_train, y_val = train_test_split(tweets.drop(['id','keyword','location','target'],axis=1), tweets[['target']], test_size=0.2, stratify=tweets[['target']], random_state=0)
 X_train_text = X_train['text']
X_val_text = X_val['text']
print('X_train shape: ', X_train.shape)
 print('X_val shape: ', X_val.shape)
 print('y_train shape: ', y_train.shape)
 print('y_val shape: ', y_val.shape )
def define_discriminator(in_shape=(28,28,1), n_classes=10):
	
	in_label = Input(shape=(1,))
	
	li = Embedding(n_classes, 50 )(in_label)
 	
	n_nodes = in_shape[0] * in_shape[1]
	li = Dense(n_nodes )(li)
 	
	li = Reshape(( in_shape[0], in_shape[1], 1))(li)
 	
	in_image = Input(shape=in_shape)
 	
	merge = Concatenate()([in_image, li])
 	
	fe = Conv2D(128,(3,3), strides=(2,2), padding='same' )(merge)
 	fe = LeakyReLU(alpha=0.2 )(fe)
 	
	fe = Conv2D(128,(3,3), strides=(2,2), padding='same' )(fe)
 	fe = LeakyReLU(alpha=0.2 )(fe)
 	
	fe = Flatten()(fe)
 	
	fe = Dropout(0.4 )(fe)
 	
	out_layer = Dense(1, activation='sigmoid' )(fe)
 	
	model = Model([in_image, in_label], out_layer)
 	
	opt = Adam(lr=0.0002, beta_1=0.5)
 	model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
 	return model<define_variables>
print('Train Class Proportion:
', y_train['target'].value_counts() / len(y_train)* 100)
 print('
Validation Class Proportion:
', y_val['target'].value_counts() / len(y_val)* 100 )
NNimage = 4096<choose_model_class>
tokenizer_1 = Tokenizer(num_words=5000, oov_token='<UNK>')
 tokenizer_1.fit_on_texts(X_train_text )
dog = Input(( NNimage,))
dogName = Input(( 10000,))
x = Dense(NNimage, activation='sigmoid' )(dogName)
 x = Reshape(( 2,NNimage,1))(concatenate([dog,x]))
x = Conv2D(1,(2,1),use_bias=False,name='conv' )(x)
 discriminated = Flatten()(x)
 discriminator = Model([dog,dogName], discriminated)
 discriminator.get_layer('conv' ).trainable = False
discriminator.get_layer('conv' ).set_weights([np.array([[[[-1.0 ]]],[[[1.0]]]])])
 discriminator.compile(optimizer='adam', loss='binary_crossentropy')
<train_model>
X_train_text = tokenizer_1.texts_to_sequences(X_train_text)
 X_val_text = tokenizer_1.texts_to_sequences(X_val_text)
 print(X_train_text[:10])
 print('')
 print(X_val_text[:10] )
train_y =(imagesIn2[:10000,:,:]/255.).reshape(( -1,NNimage))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,NNimage))
lr = 0.5
for k in range(4):
annealer = LearningRateScheduler(lambda x: lr)
 h = discriminator.fit([zeros,train_X], train_y, epochs = 20, batch_size=256, callbacks=[annealer], verbose=0)
 print('Epoch',(k+1)*10,'/50 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1]<0.530: lr = 0.1<compute_test_metric>
tokenizer_1.sequences_to_texts([X_train_text[1]] )
print('Epoch',(k+1)*10,'/50 - loss =',h.history['loss'][-1] )<choose_model_class>
vocab_size = len(tokenizer_1.word_index)+ 1
embeddings_index = dict()
f = open('.. /input/glove-global-vectors-for-word-representation/glove.twitter.27B.200d.txt')
 for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Loaded %s word vectors.' % len(embeddings_index))
def define_generator(latent_dim):
	model = Sequential()
	
	n_nodes = 128 * 7 * 7
	model.add(Dense(n_nodes, input_dim=latent_dim))
	model.add(LeakyReLU(alpha=0.2))
	model.add(Reshape(( 7, 7, 128)))
 	
	model.add(Conv2DTranspose(128,(4,4), strides=(2,2), padding='same'))
	model.add(LeakyReLU(alpha=0.2))
	
	model.add(Conv2DTranspose(128,(4,4), strides=(2,2), padding='same'))
	model.add(LeakyReLU(alpha=0.2))
	
	model.add(Conv2D(1,(7,7), activation='tanh', padding='same'))
	return model<categorify>
embedding_matrix = np.zeros(( vocab_size, 200))
for word, i in tokenizer_1.word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print('Embedding Matrix Shape:', embedding_matrix.shape )
def define_generator(latent_dim, n_classes=120):
	
	in_label = Input(shape=(1,))
	
	li = Embedding(n_classes, 50 )(in_label)
 	
	n_nodes = 7 * 7
	li = Dense(n_nodes )(li)
 	
	li = Reshape(( 7, 7, 1))(li)
 	
	in_lat = Input(shape=(latent_dim,))
	
	n_nodes = 128 * 7 * 7
	gen = Dense(n_nodes )(in_lat)
 	gen = LeakyReLU(alpha=0.2 )(gen)
 	gen = Reshape(( 7, 7, 128))(gen)
 	
	merge = Concatenate()([gen, li])
 	
	gen = Conv2DTranspose(128,(4,4), strides=(2,2), padding='same' )(merge)
 	gen = LeakyReLU(alpha=0.2 )(gen)
 	
	gen = Conv2DTranspose(128,(4,4), strides=(2,2), padding='same' )(gen)
 	gen = LeakyReLU(alpha=0.2 )(gen)
 	
	out_layer = Conv2D(1,(7,7), activation='tanh', padding='same' )(gen)
 	
	model = Model([in_lat, in_label], out_layer)
 	return model<choose_model_class>
num_epochs=15
dropout=0.2
recurrent_dropout=0.2
lr=0.0005
batch_size=128
class_weight = {0: y_train['target'].value_counts() [1]/len(y_train), 1: y_train['target'].value_counts() [0]/len(y_train)}
discriminator.trainable=False
gan_input = Input(shape=(10000,))
x = generator(gan_input)
 gan_output = discriminator(x)
 gan = Model(gan_input, gan_output)
 gan.get_layer('model_1' ).get_layer('conv' ).set_weights([np.array([[[[-1 ]]],[[[255.]]]])])
 gan.compile(optimizer=Adam(5), loss='mean_squared_error')
<train_on_grid>
lstm_model = Sequential()
embedding_layer = Embedding(vocab_size, 200, weights=[embedding_matrix], input_length=maxlen, trainable=False)
 lstm_model.add(embedding_layer)
 lstm_model.add(LSTM(128, return_sequences=True, dropout=dropout, recurrent_dropout=recurrent_dropout))
lstm_model.add(LSTM(128))
lstm_model.add(Dense(1, activation='sigmoid'))
adam = optimizers.Adam(lr=lr)
 lstm_model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc'])
 print(lstm_model.summary())
train = np.zeros(( 10000,10000))
for i in range(10000): train[i,i] = 1
zeros = np.zeros(( 10000,NNimage))
lr = 5.
 for k in range(20):
annealer = LearningRateScheduler(lambda x: lr)
 h = gan.fit(train, zeros, epochs = 1, batch_size=256, callbacks=[annealer], verbose=0)
 if(k<10)|(k%5==4):
print('Epoch',(k+1)*10,'/500 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1] < 25: lr = 1.
 if h.history['loss'][-1] < 1.5: lr = 0.5
if k<10:
plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 img = generator.predict(xx.reshape(( -1,10000)))[0]
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64)))
 plt.axis('off')
 plt.imshow(img, cmap='Greys')
 plt.show()<import_modules>
checkpoint = ModelCheckpoint('lstm_model.h5', monitor='val_acc', save_best_only=True)
 history = lstm_model.fit(X_train_text, y_train, batch_size=batch_size, callbacks=[checkpoint], epochs=num_epochs,
class_weight=class_weight, validation_data=(X_val_text, y_val), verbose=1)
 plot_model_performance(history )
warnings.filterwarnings('ignore', category=UserWarning, module='skimage')
 seed = 42
random.seed = seed
np.random.seed = seed 
<define_variables>
IMG_WIDTH = 64
IMG_HEIGHT = 64
IMG_CHANNELS = 3
INPUT_SHAPE=(IMG_HEIGHT, IMG_WIDTH, 1)
<categorify>
class attention(Layer):
def __init__(self,**kwargs):
super(attention,self ).__init__(**kwargs)
 def build(self,input_shape):
self.W=self.add_weight(name="att_weight",shape=(input_shape[-1],1),initializer="normal")
 self.b=self.add_weight(name="att_bias",shape=(input_shape[1],1),initializer="zeros")
 super(attention, self ).build(input_shape)
 def call(self,x):
et=K.squeeze(K.tanh(K.dot(x,self.W)+self.b),axis=-1)
 at=K.softmax(et)
 at=K.expand_dims(at,axis=-1)
 output=x*at
return K.sum(output,axis=1)
 def compute_output_shape(self,input_shape):
return(input_shape[0],input_shape[-1])
 def get_config(self):
return super(attention,self ).get_config()
L_enc = sklearn.preprocessing.LabelEncoder()
labels = L_enc.fit_transform(namesIn )<categorify>
num_epochs=15
dropout=0.3
recurrent_dropout=0.3
lr=0.0005
batch_size=128
sequence_input = Input(shape=(maxlen,))
embedded_sequences = Embedding(vocab_size, 200, weights=[embedding_matrix], trainable=False )(sequence_input)
 lstm = Bidirectional(LSTM(128, return_sequences = True, dropout=dropout, recurrent_dropout=recurrent_dropout))(embedded_sequences)
(lstm, forward_h, forward_c, backward_h, backward_c)= Bidirectional(LSTM(128, return_sequences=True, return_state=True))(lstm)
 att_out=attention()(lstm)
 outputs=Dense(1,activation='sigmoid' )(att_out)
 model_attn = Model(sequence_input, outputs)
 adam = optimizers.Adam(lr=lr)
 model_attn.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc'])
 print(model_attn.summary() )
def Colorize() :
in_label = Input(shape=(1,))
embed_input = Embedding(120, 1000 )(in_label)
 encoder_input = Input(shape=(IMG_WIDTH, IMG_HEIGHT, 1,))
encoder_output = Conv2D(32,(3,3), activation='relu', padding='same',strides=1 )(encoder_input)
 encoder_output = MaxPooling2D(( 2, 2), padding='same' )(encoder_output)
 encoder_output = Conv2D(32,(4,4), activation='relu', padding='same' )(encoder_output)
 encoder_output = Conv2D(32,(3,3), activation='relu', padding='same',strides=1 )(encoder_output)
 encoder_output = MaxPooling2D(( 2, 2), padding='same' )(encoder_output)
 encoder_output = Conv2D(64,(4,4), activation='relu', padding='same' )(encoder_output)
 encoder_output = Conv2D(64,(3,3), activation='relu', padding='same',strides=1 )(encoder_output)
 encoder_output = MaxPooling2D(( 2, 2), padding='same' )(encoder_output)
 encoder_output = Conv2D(64,(4,4), activation='relu', padding='same' )(encoder_output)
 encoder_output = Conv2D(64,(3,3), activation='relu', padding='same' )(encoder_output)
 encoder_output = Conv2D(64,(3,3), activation='relu', padding='same' )(encoder_output)
 fusion_output = Dense(4096, activation='relu' )(embed_input)
 fusion_output = Reshape(( [8, 8, 64]))(fusion_output)
 fusion_output = concatenate([encoder_output, fusion_output], axis=3)
 fusion_output = Conv2D(64,(1, 1), activation='relu', padding='same' )(fusion_output)
 decoder_output = Conv2D(64,(3,3), activation='relu', padding='same' )(fusion_output)
 decoder_output = Conv2D(32,(3,3), activation='relu', padding='same' )(decoder_output)
 decoder_output = UpSampling2D(( 2, 2))(decoder_output)
 decoder_output = Conv2D(64,(3,3), activation='relu', padding='same' )(decoder_output)
 decoder_output = UpSampling2D(( 2, 2))(decoder_output)
 decoder_output = Conv2D(128,(4,4), activation='relu', padding='same' )(decoder_output)
 decoder_output = Conv2D(128,(3,3), activation='relu', padding='same' )(decoder_output)
 decoder_output = Conv2D(32,(2,2), activation='relu', padding='same' )(decoder_output)
 decoder_output = Conv2D(3,(3, 3), activation='tanh', padding='same' )(decoder_output)
 decoder_output = UpSampling2D(( 2, 2))(decoder_output)
 return Model(inputs=[encoder_input, in_label], outputs=decoder_output)
 model = Colorize()
model.compile(optimizer='adam', loss='mean_squared_error')
 model.summary()<prepare_x_and_y>
checkpoint = ModelCheckpoint('attn_model.h5', monitor='val_acc', save_best_only=True)
 history_attn = model_attn.fit(X_train_text, y_train, batch_size=batch_size, callbacks=[checkpoint], epochs=num_epochs,
class_weight=class_weight, validation_data=(X_val_text, y_val), verbose=1)
 plot_model_performance(history_attn )
X_train = imagesIn / 256 .<init_hyperparams>
maxlen = 160
lr = 1e-5
num_epochs = 5
batch_size=16
datagen = ImageDataGenerator(
 shear_range=0.2,
zoom_range=0.2,
rotation_range=20,
horizontal_flip=True)
 def image_a_b_gen(dataset=X_train, batch_size = 20):
for batch in datagen.flow(dataset, batch_size=batch_size):
X_batch = rgb2gray(batch)
 lab_batch = rgb2lab(batch)
 X_batch = lab_batch[:,:,:,0]
X_batch = X_batch.reshape(X_batch.shape+(1,))
Y_batch = lab_batch[:,:,:,1:] / 128
yield [X_batch, label_batch], Y_batch
learning_rate_reduction = ReduceLROnPlateau(monitor='loss',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 filepath = "Art_Colorization_Model.h5"
checkpoint = ModelCheckpoint(filepath,
save_best_only=True,
monitor='loss',
mode='min')
 model_callbacks = [learning_rate_reduction,checkpoint]<train_model>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
model.fit([X_batch,labels],X_train,
epochs=6,
verbose=1 )<categorify>
def build_model(bert_layer, max_len=512, lr=1e-5):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=lr), loss='binary_crossentropy', metrics=['accuracy'])
 return model
class DogGenerator:
index = 0
def getDog(self,seed):
xx = np.zeros(( 10000))
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( 64,64))
self.index =(self.index+1)%10000
return Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64)))
<import_modules>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
 NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 3
TOP_K = 3
MAX_INLIER_SCORE = 35
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 8000000
HOMOGRAPHY_CONFIDENCE = 0.99
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'])
 def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,
DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0,
name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
 def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 main()<import_modules>
train_input = bert_encode(X_train.text.values, tokenizer, max_len=maxlen)
 val_input = bert_encode(X_val.text.values, tokenizer, max_len=maxlen)
 train_labels = y_train.target.values
val_labels = y_val.target.values
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
 NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 3
TOP_K = 3
MAX_INLIER_SCORE = 35
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 8500000
HOMOGRAPHY_CONFIDENCE = 0.99
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'])
 def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,
DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0,
name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
 def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 main()<load_from_csv>
bert_model = build_model(bert_layer, max_len=maxlen, lr=lr)
 bert_model.summary()
train = pd.read_csv('.. /input/train.csv' )<drop_column>
checkpoint = ModelCheckpoint('bertmodel.h5', monitor='val_accuracy', save_best_only=True)
 bert_history = bert_model.fit(
 train_input, train_labels,
validation_data=(val_input, val_labels),
epochs=num_epochs,
callbacks=[checkpoint],
batch_size=batch_size
)
x_train = train.drop(['id' ,'target'] ,axis = 1 )<load_from_csv>
clf = RandomForestClassifier(n_estimators=500, max_depth=15, min_samples_split=20, min_samples_leaf=2, n_jobs=-1, random_state=0)
 clf.fit(X_train.drop('text',axis=1), y_train.target.values)
 clf_pred = clf.predict_proba(X_val.drop('text',axis=1))
print('Validation Accuracy:', accuracy_score(y_val.target.values, clf_pred.argmax(axis=-1)) )
test = pd.read_csv('.. /input/test.csv' )<drop_column>
bert_model.load_weights('bertmodel.h5')
 val = X_val.copy()
val = val[['text']]
val['target'] = y_val
val['pred'] = bert_model.predict(val_input)
 val['pred'] = val['pred'].apply(lambda x: 1 if x >=0.5 else 0)
 error = val[val['target'] != val['pred']]
error.head()
test = test.drop('id',axis = 1 )<normalization>
test['char_len'] = test.text.str.len()
word_tokens = [len(word_tokenize(tweet)) for tweet in test.text]
test['word_len'] = word_tokens
sent_tokens = [len(sent_tokenize(tweet)) for tweet in test.text]
test['sent_len'] = sent_tokens
scaler = StandardScaler()
x_train = scaler.fit_transform(x_train)
 test = scaler.fit_transform(test )<train_on_grid>
test['polarity'] = [TextBlob(tweet ).sentiment.polarity for tweet in test.text]
test['subjectivity'] = [TextBlob(tweet ).sentiment.subjectivity for tweet in test.text]
test['exclaimation_num'] = [tweet.count('!')for tweet in test.text]
test['questionmark_num'] = [tweet.count('?')for tweet in test.text]
def count_url_hashtag_mention(text):
urls_num = len(re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', text))
word_tokens = text.split()
hash_num = len([word for word in word_tokens if word[0] == '
mention_num = len([word for word in word_tokens if word[0] == '@' and word.count('@')== 1])
 return urls_num, hash_num, mention_num
url_num, hash_num, mention_num = zip(*[count_url_hashtag_mention(tweet)for tweet in test.text])
 test = test.assign(url_num = url_num, hash_num = hash_num, mention_num = mention_num)
 contractions = ["'t", "'re", "'s", "'d", "'ll", "'ve", "'m"]
test['contraction_num'] = [sum([tweet.count(cont)for cont in contractions])for tweet in test.text]
model_lasso = LassoCV(alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75] ).fit(x_train, y_train )<predict_on_test>
test.keyword.fillna('None', inplace=True)
 def decontraction(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
test.text = [decontraction(tweet)for tweet in test.text]
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 print(remove_emoji("OMG there is a volcano eruption!!! 😭😱😷"))
test.text = test.text.apply(lambda x: remove_emoji(x))
lasso_preds = model_lasso.predict(test )<prepare_output>
test.text = test.text.apply(lambda x: remove_url(x))
def remove_punct(text):
new_punct = re.sub('\ |\!|\?', '', punctuation)
 table=str.maketrans('','',new_punct)
 return text.translate(table)
 test.text = test.text.apply(lambda x: remove_punct(x))
def replace_amp(text):
text = re.sub(r" amp ", " and ", text)
 return text
test.text = test.text.apply(lambda x: replace_amp(x))
lasso_preds=lasso_preds.T<load_from_csv>
test = pd.read_csv('.. /input/twitter-logo/tweets_test_segmented.csv' )
lasso_preds
test = pd.read_csv('.. /input/test.csv')
<save_to_csv>
lemmatizer = WordNetLemmatizer()
def lemma(text):
words = word_tokenize(text)
 return ' '.join([lemmatizer.lemmatize(w.lower() , pos='v')for w in words])
 test.text = test.text.apply(lambda x: lemma(x))
solution = pd.DataFrame({"id":test.id, "target":lasso_preds})
 solution.to_csv("lasso_sol.csv", index = False )<feature_engineering>
test_text = test['text']
test_text = tokenizer_1.texts_to_sequences(test_text)
 test_text = pad_sequences(test_text, padding='post', maxlen=50)
 print('X_test shape:', test_text.shape )
np.random.seed(300)
 data = pd.DataFrame(np.zeros(( 20000,300)))
 for i in range(300): data.iloc[:,i] = np.random.normal(0,1,20000)
 important = 35; noise = 3.5
a = np.random.normal(0,1,300)
 x = np.random.choice(np.arange(300),300-important,replace=False); a[x] = 0
data['target'] = data.values.dot(a)+ np.random.normal(0,noise,20000)
 data.sort_values('target',inplace=True)
 data.iloc[:7200,300] = 0.0
data.iloc[7200:,300] = 1.0
train = data.sample(250)
 public = data[ ~data.index.isin(train.index)].sample(1975)
 private = data[ ~data.index.isin(train.index)& ~data.index.isin(public.index)].sample(frac=1)
 train.reset_index(drop=True,inplace=True)
 public.reset_index(drop=True,inplace=True)
 private.reset_index(drop=True,inplace=True )<train_model>
lstm_model.load_weights('lstm_model.h5')
 submission = test.copy() [['id']]
submission['target'] = lstm_model.predict_classes(test_text)
 submission.to_csv('submission.csv', index=False)
 display(submission.head())
oof = np.zeros(len(train))
predsPU= np.zeros(len(public))
predsPR= np.zeros(len(private))
rskf = RepeatedStratifiedKFold(n_splits=25, n_repeats=5)
 for train_index, test_index in rskf.split(train.iloc[:,:-1], train['target']):
clf = LogisticRegression(solver='liblinear',penalty='l2',C=1.0,class_weight='balanced')
 clf.fit(train.loc[train_index].iloc[:,:-1],train.loc[train_index]['target'])
 oof[test_index] += clf.predict_proba(train.loc[test_index].iloc[:,:-1])[:,1]
predsPU += clf.predict_proba(public.iloc[:,:-1])[:,1]
predsPR += clf.predict_proba(private.iloc[:,:-1])[:,1]
aucTR = round(roc_auc_score(train['target'],oof),5)
 aucPU = round(roc_auc_score(public['target'],predsPU),5)
 aucPR = round(roc_auc_score(private['target'],predsPR),5)
 print('LR Model with L2-penalty: CV =',aucTR,'LB =',aucPU,'Private score =',aucPR )<find_best_model_class>
model_attn.load_weights('attn_model.h5')
 submission_attn = test.copy() [['id']]
submission_attn['target'] = model_attn.predict(test_text)
 submission_attn['target'] = submission_attn['target'].apply(lambda x: 1 if x >=0.5 else 0)
 submission_attn.to_csv('submission_attn.csv', index=False)
 display(submission_attn.head() )
oof = np.zeros(len(train))
predsPU= np.zeros(len(public))
predsPR= np.zeros(len(private))
rskf = RepeatedStratifiedKFold(n_splits=25, n_repeats=5)
 for train_index, test_index in rskf.split(train.iloc[:,:-1], train['target']):
clf = LogisticRegression(solver='liblinear',penalty='l1',C=0.1,class_weight='balanced')
 clf.fit(train.loc[train_index].iloc[:,:-1],train.loc[train_index]['target'])
 oof[test_index] += clf.predict_proba(train.loc[test_index].iloc[:,:-1])[:,1]
predsPU += clf.predict_proba(public.iloc[:,:-1])[:,1]
predsPR += clf.predict_proba(private.iloc[:,:-1])[:,1]
aucTR = round(roc_auc_score(train['target'],oof),5)
 aucPU = round(roc_auc_score(public['target'],predsPU),5)
 aucPR = round(roc_auc_score(private['target'],predsPR),5)
 print('LR Model with L1-penalty: CV =',aucTR,'LB =',aucPU,'Private score =',aucPR )<train_on_grid>
test_input = bert_encode(test.text.values, tokenizer, max_len=100)
 bert_model.load_weights('bertmodel.h5')
 submission_bert = test.copy() [['id']]
submission_bert['target'] = bert_model.predict(test_input)
 submission_bert['target'] = submission_bert['target'].apply(lambda x: 1 if x >=0.5 else 0)
 submission_bert.to_csv('submission_bert.csv', index=False)
 display(submission_bert.head() )
<feature_engineering><EOS> 
clf_testpred = clf.predict_proba(test.drop(['id','keyword','location','text'],axis=1))
submission_bert = test.copy() [['id']]
submission_bert['target'] =(bert_model.predict(test_input)*0.8 ).ravel() +(clf_testpred.max(axis=1)*0.2)
 submission_bert['target'] = submission_bert['target'].apply(lambda x: 1 if x >=0.5 else 0)
 submission_bert.to_csv('submission_bert_ensemble.csv', index=False)
 display(submission_bert.head() )