Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<prepare_x_and_y>
nlp = spacy.load("en_core_web_lg" )
idx = np.random.randint(0,idxIn,10000)
 train_y = imagesIn[idx,:,:,:]/255.
 train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1<categorify>
train_data = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test_data =pd.read_csv('.. /input/nlp-getting-started/test.csv' )
lr = 0.005
for k in range(50):
annealer = LearningRateScheduler(lambda x: lr)
 h = decoder.fit(train_X, train_y, epochs = 10, batch_size=256, callbacks=[annealer], verbose=0)
 if k%5==4: print('Epoch',(k+1)*10,'/500 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1]<0.54: lr = 0.001<drop_column>
def clean_text(text):
url = re.compile(r'https?://\S+|www\.\S+')
 text = url.sub(r'', text)
 html = re.compile(r'<.*?>')
 text = html.sub(r'', text)
 emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags = re.UNICODE)
 text = emoji_pattern.sub(r'', text)
 table = str.maketrans('', '', string.punctuation)
 text = text.translate(table)
 text = text.lower()
text = re.sub(r"i'm", "i am", text)
 text = re.sub(r"you'll", "you will", text)
 text = re.sub(r"i'll", "i will", text)
 text = re.sub(r"she'll", "she will", text)
 text = re.sub(r"he'll", "he will", text)
 text = re.sub(r"he's", "he is", text)
 text = re.sub(r"she's", "she is", text)
 text = re.sub(r"that's", "that is", text)
 text = re.sub(r"what's", "what is", text)
 text = re.sub(r"where's", "where is", text)
 text = re.sub(r"there's", "there is", text)
 text = re.sub(r"here's", "here is", text)
 text = re.sub(r"who's", "who is", text)
 text = re.sub(r"how's", "how is", text)
 text = re.sub(r"'ll", " will", text)
 text = re.sub(r"'ve", " have", text)
 text = re.sub(r"'re", " are", text)
 text = re.sub(r"'d", " would", text)
 text = re.sub(r"can't", "cannot", text)
 text = re.sub(r"won't", "will not", text)
 text = re.sub(r"don't", "do not", text)
 text = re.sub(r"shouldn't", "should not", text)
 text = re.sub(r"n't", " not", text)
 text = re.sub(r" ", " ", text)
 text = "".join([c if c not in string.punctuation else "" for c in text])
 text = ''.join(c for c in text if not c.isdigit())
 text = re.sub(r"÷", "", text)
 text = re.sub(' +', ' ', text)
 return text
train_data['text'] = train_data['text'].apply(lambda x : clean_text(x))
test_data['text'] = test_data['text'].apply(lambda x : clean_text(x))
del train_X, train_y, imagesIn<categorify>
def massage_text(text):
text = re.sub("[^a-zA-Z]", ' ', text)
 text = text.lower()
text = text.split()
lem = WordNetLemmatizer()
text = [lem.lemmatize(word)for word in text
if word not in set(stopwords.words('english')) ]
text = ' '.join(text)
 return text
print('--here goes nothing')
 print(text)
 train_data['text'] = train_data['text'].apply(lambda x : massage_text(x))
test_data['text'] = test_data['text'].apply(lambda x : massage_text(x))
print('Generate Random Dogs')
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
xx[np.random.randint(10000)] = 0.75
xx = xx/(np.sqrt(xx.dot(xx.T)))
 plt.subplot(1,5,j+1)
 img = decoder.predict(xx.reshape(( -1,10000)))
 img = Image.fromarray(( 255*img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()<categorify>
vectorizer = TfidfVectorizer()
Train = vectorizer.fit_transform(train_data['text'])
 test = vectorizer.transform(test_data['text'] )
print('Recall from Memory Dogs')
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 img = decoder.predict(xx.reshape(( -1,10000)))
 img = Image.fromarray(( 255*img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()<categorify>
pbf = linear_model.RidgeClassifier(alpha=2.2)
 pbf.fit(Train, train_data["target"])
 scores = model_selection.cross_val_score(pbf, Train, train_data["target"], cv=3, scoring="f1")
 scores
z = zipfile.PyZipFile('images.zip', mode='w')
 for k in range(10000):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 0.99
xx[np.random.randint(10000)] = 0.01
img = decoder.predict(xx.reshape(( -1,10000)))
 img = Image.fromarray(( 255*img ).astype('uint8' ).reshape(( 64,64,3)))
 f = str(k)+'.png'
img.save(f,'PNG'); z.write(f); os.remove(f)
 z.close()<choose_model_class>
score = scores.mean() *100
print("Score: ", score )
class KernelEvalException(Exception):
pass
model_params = {
'Inception': {
'name': 'Inception',
'imsize': 64,
'output_layer': 'Pretrained_Net/pool_3:0',
'input_layer': 'Pretrained_Net/ExpandDims:0',
'output_shape': 2048,
'cosine_distance_eps': 0.1
}
}
def create_model_graph(pth):

 with tf.gfile.FastGFile(pth, 'rb')as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
 _ = tf.import_graph_def(graph_def, name='Pretrained_Net')
 def _get_model_layer(sess, model_name):
layername = model_params[model_name]['output_layer']
layer = sess.graph.get_tensor_by_name(layername)
 ops = layer.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
if shape._dims != []:
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
 else:
new_shape.append(s)
 o.__dict__['_shape_val'] = tf.TensorShape(new_shape)
 return layer
def get_activations(images, sess, model_name, batch_size=50, verbose=False):

 inception_layer = _get_model_layer(sess, model_name)
 n_images = images.shape[0]
if batch_size > n_images:
print("warning: batch size is bigger than the data size.setting batch size to data size")
 batch_size = n_images
n_batches = n_images//batch_size + 1
pred_arr = np.empty(( n_images,model_params[model_name]['output_shape']))
for i in tqdm(range(n_batches)) :
if verbose:
print("\rPropagating batch %d/%d" %(i+1, n_batches), end="", flush=True)
 start = i*batch_size
if start+batch_size < n_images:
end = start+batch_size
else:
end = n_images
batch = images[start:end]
pred = sess.run(inception_layer, {model_params[model_name]['input_layer']: batch})
 pred_arr[start:end] = pred.reshape(-1,model_params[model_name]['output_shape'])
 if verbose:
print(" done")
 return pred_arr
def normalize_rows(x: np.ndarray):

 return np.nan_to_num(x/np.linalg.norm(x, ord=2, axis=1, keepdims=True))
def cosine_distance(features1, features2):
features1_nozero = features1[np.sum(features1, axis=1)!= 0]
features2_nozero = features2[np.sum(features2, axis=1)!= 0]
norm_f1 = normalize_rows(features1_nozero)
 norm_f2 = normalize_rows(features2_nozero)
 d = 1.0-np.abs(np.matmul(norm_f1, norm_f2.T))
print('d.shape=',d.shape)
 print('np.min(d, axis=1 ).shape=',np.min(d, axis=1 ).shape)
 mean_min_d = np.mean(np.min(d, axis=1))
print('distance=',mean_min_d)
 return mean_min_d
def distance_thresholding(d, eps):
if d < eps:
return d
else:
return 1
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):

 mu1 = np.atleast_1d(mu1)
 mu2 = np.atleast_1d(mu2)
 sigma1 = np.atleast_2d(sigma1)
 sigma2 = np.atleast_2d(sigma2)
 assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
 if not np.isfinite(covmean ).all() :
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
 offset = np.eye(sigma1.shape[0])* eps
covmean = linalg.sqrtm(( sigma1 + offset ).dot(sigma2 + offset))
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean ).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
print('covmean.shape=',covmean.shape)
 tr_covmean = np.trace(covmean)
 return diff.dot(diff)+ np.trace(sigma1)+ np.trace(sigma2)- 2 * tr_covmean
def calculate_activation_statistics(images, sess, model_name, batch_size=50, verbose=False):

 act = get_activations(images, sess, model_name, batch_size, verbose)
 mu = np.mean(act, axis=0)
 sigma = np.cov(act, rowvar=False)
 return mu, sigma, act
def _handle_path_memorization(path, sess, model_name, is_checksize, is_check_png):
path = pathlib.Path(path)
 files = list(path.glob('*.jpg')) + list(path.glob('*.png'))
imsize = model_params[model_name]['imsize']
x = np.array([np.array(img_read_checks(fn, imsize, is_checksize, imsize, is_check_png)) for fn in files])
 m, s, features = calculate_activation_statistics(x, sess, model_name)
 del x
return m, s, features
def img_read_checks(filename, resize_to, is_checksize=False, check_imsize = 64, is_check_png = False):
im = Image.open(str(filename))
if is_checksize and im.size !=(check_imsize,check_imsize):
raise KernelEvalException('The images are not of size '+str(check_imsize))
if is_check_png and im.format != 'PNG':
raise KernelEvalException('Only PNG images should be submitted.')
 if resize_to is None:
return im
else:
return im.resize(( resize_to,resize_to),Image.ANTIALIAS)
 def calculate_kid_given_paths(paths, model_name, model_path, feature_path=None, mm=[], ss=[], ff=[]):

 tf.reset_default_graph()
create_model_graph(str(model_path))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
 m1, s1, features1 = _handle_path_memorization(paths[0], sess, model_name, is_checksize = True, is_check_png = True)
 if len(mm)!= 0:
m2 = mm
s2 = ss
features2 = ff
elif feature_path is None:
m2, s2, features2 = _handle_path_memorization(paths[1], sess, model_name, is_checksize = False, is_check_png = False)
 else:
with np.load(feature_path)as f:
m2, s2, features2 = f['m'], f['s'], f['features']
print('m1,m2 shape=',(m1.shape,m2.shape),'s1,s2=',(s1.shape,s2.shape))
print('starting calculating FID')
 fid_value = calculate_frechet_distance(m1, s1, m2, s2)
 print('done with FID, starting distance calculation')
 distance = cosine_distance(features1, features2)
 return fid_value, distance, m2, s2, features2<load_pretrained>
pbf.fit(Train, train_data["target"] )
if ComputeLB:
with zipfile.ZipFile(".. /working/images.zip","r")as z:
z.extractall(".. /tmp/images2/")
 m2 = []; s2 =[]; f2 = []
user_images_unzipped_path = '.. /tmp/images2/'
images_path = [user_images_unzipped_path,'.. /input/generative-dog-images/all-dogs/all-dogs/']
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
fid_epsilon = 10e-15
fid_value_public, distance_public, m2, s2, f2 = calculate_kid_given_paths(images_path, 'Inception', public_path, mm=m2, ss=s2, ff=f2)
 distance_public = distance_thresholding(distance_public, model_params['Inception']['cosine_distance_eps'])
 print("FID_public: ", fid_value_public, "distance_public: ", distance_public, "multiplied_public: ",
fid_value_public /(distance_public + fid_epsilon))
! rm -r.. /tmp<import_modules>
pbf.predict(test )
batch_size = 8<define_variables>
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = pbf.predict(test)
 sample_submission.head(20 )
<set_options><EOS> 
sample_submission.to_csv("submission.csv", index=False )