Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<sort_values>
import pandas as pd
train_features = train_features.sort_values(
 by=["sig_id"], axis=0, inplace=False ).reset_index(drop=True)
 train_labels = train_labels.sort_values(by=["sig_id"], axis=0,
inplace=False ).reset_index(drop=True)
 train_extra_labels = train_extra_labels.sort_values(
 by=["sig_id"], axis=0, inplace=False ).reset_index(drop=True)
 sample_submission = sample_submission.sort_values(
 by=["sig_id"], axis=0, inplace=False ).reset_index(drop=True )<define_variables>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 print(train.shape)
 train.head(5 )
category_features = ["cp_type", "cp_dose"]
numeric_features = [
c for c in train_features.columns
if c != "sig_id" and c not in category_features
]
all_features = category_features + numeric_features
gene_experssion_features = [c for c in numeric_features if c.startswith("g-")]
cell_viability_features = [c for c in numeric_features if c.startswith("c-")]
len(numeric_features), len(gene_experssion_features), len(
 cell_viability_features )<define_variables>
test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 print(test.shape)
 test.head(5 )
train_classes = [c for c in train_labels.columns if c != "sig_id"]
train_extra_classes = [c for c in train_extra_labels.columns if c != "sig_id"]
len(train_classes), len(train_extra_classes )<categorify>
import re
for df in [train_features, test_features]:
df['cp_type'] = df['cp_type'].map({'ctl_vehicle': 0, 'trt_cp': 1})
 df['cp_dose'] = df['cp_dose'].map({'D1': 0, 'D2': 1})
 df['cp_time'] = df['cp_time'].map({24: 0, 48: 0.5, 72: 1} )<count_values>
def urlremove(text):
return re.sub(r"(?:\@|https?\://)\S+", "", text)
 train['text'] = train['text'].apply(urlremove)
 test['text'] = test['text'].apply(urlremove)
 train['text']
train_features["cp_type"].value_counts()<count_values>
def remove_emoji(text):
emoji_pattern = re.compile(
 u"(\ud83d[\ude00-\ude4f])|"
u"(\ud83c[\udf00-\uffff])|"
u"(\ud83d[\u0000-\uddff])|"
u"(\ud83d[\ude80-\udeff])|"
u"(\ud83c[\udde0-\uddff])"
"+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train['text'] = train['text'].apply(remove_emoji)
 test['text'] = test['text'].apply(remove_emoji)
 train['text']
train_features["cp_dose"].value_counts()<count_values>
def decontracted(text):
if '’' in text:
text = text.replace("’", "'")
 text = re.sub(r"n't", " not", text)
 text = re.sub(r"'re", " are", text)
 text = re.sub(r"'s", " is", text)
 text = re.sub(r"'d", " would", text)
 text = re.sub(r"'ll", " will", text)
 text = re.sub(r"'t", " not", text)
 text = re.sub(r"'ve", " have", text)
 text = re.sub(r"'m", " am", text)
 return text
train['text'] = train['text'].apply(decontracted)
 test['text'] = test['text'].apply(decontracted)
 train['text']
train_features["cp_time"].value_counts()<import_modules>
def character_remove(text):
return re.sub('\W+|_', ' ', text)
 train.text = train.text.apply(character_remove)
 test.text = test.text.apply(character_remove)
 train.text
class DeepInsightTransformer:

 def __init__(self,
feature_extractor='tsne',
perplexity=30,
pixels=100,
random_state=None,
n_jobs=None):

 self.random_state = random_state
self.n_jobs = n_jobs
if isinstance(feature_extractor, str):
fe = feature_extractor.casefold()
if fe == 'tsne_exact'.casefold() :
fe = TSNE(n_components=2,
metric='cosine',
perplexity=perplexity,
n_iter=1000,
method='exact',
random_state=self.random_state,
n_jobs=self.n_jobs)
 elif fe == 'tsne'.casefold() :
fe = TSNE(n_components=2,
metric='cosine',
perplexity=perplexity,
n_iter=1000,
method='barnes_hut',
random_state=self.random_state,
n_jobs=self.n_jobs)
 elif fe == 'pca'.casefold() :
fe = PCA(n_components=2, random_state=self.random_state)
 elif fe == 'kpca'.casefold() :
fe = KernelPCA(n_components=2,
kernel='rbf',
random_state=self.random_state,
n_jobs=self.n_jobs)
 else:
raise ValueError(( "Feature extraction method '{}' not accepted"
).format(feature_extractor))
self._fe = fe
elif hasattr(feature_extractor, 'fit_transform')and \
inspect.ismethod(feature_extractor.fit_transform):
self._fe = feature_extractor
else:
raise TypeError('Parameter feature_extractor is not a '
'string nor has method "fit_transform"')
 if isinstance(pixels, int):
pixels =(pixels, pixels)
 self._pixels = pixels
self._xrot = None
def fit(self, X, y=None, plot=False):

 X = X.T
x_new = self._fe.fit_transform(X)
 chvertices = ConvexHull(x_new ).vertices
hull_points = x_new[chvertices]
mbr, mbr_rot = self._minimum_bounding_rectangle(hull_points)
 self._xrot = np.dot(mbr_rot, x_new.T ).T
self._calculate_coords()
if plot is True:
fig, ax = plt.subplots(1, 1, figsize=(10, 7), squeeze=False)
 ax[0, 0].scatter(x_new[:, 0],
x_new[:, 1],
cmap=plt.cm.get_cmap("jet", 10),
marker="x",
alpha=1.0)
 ax[0, 0].fill(x_new[chvertices, 0],
x_new[chvertices, 1],
edgecolor='r',
fill=False)
 ax[0, 0].fill(mbr[:, 0], mbr[:, 1], edgecolor='g', fill=False)
 plt.gca().set_aspect('equal', adjustable='box')
 plt.show()
return self
@property
def pixels(self):

 return self._pixels
@pixels.setter
def pixels(self, pixels):

 if isinstance(pixels, int):
pixels =(pixels, pixels)
 self._pixels = pixels
if hasattr(self, '_coords'):
self._calculate_coords()
def _calculate_coords(self):

 ax0_coord = np.digitize(self._xrot[:, 0],
bins=np.linspace(min(self._xrot[:, 0]),
max(self._xrot[:, 0]),
self._pixels[0])) - 1
ax1_coord = np.digitize(self._xrot[:, 1],
bins=np.linspace(min(self._xrot[:, 1]),
max(self._xrot[:, 1]),
self._pixels[1])) - 1
self._coords = np.stack(( ax0_coord, ax1_coord))
def transform(self, X, empty_value=0):

 img_coords = pd.DataFrame(np.vstack(
(self._coords, X.clip(0, 1)) ).T ).groupby(
 [0, 1],
as_index=False ).mean()
img_matrices = []
blank_mat = np.zeros(self._pixels)
 if empty_value != 0:
blank_mat[:] = empty_value
for z in range(2, img_coords.shape[1]):
img_matrix = blank_mat.copy()
img_matrix[img_coords[0].astype(int),
img_coords[1].astype(int)] = img_coords[z]
img_matrices.append(img_matrix)
 return img_matrices
def fit_transform(self, X, empty_value=0):

 self.fit(X)
 return self.transform(X, empty_value=empty_value)
 def feature_density_matrix(self):

 fdmat = np.zeros(self._pixels)
 coord_cnt =(
 pd.DataFrame(self._coords.T ).assign(count=1 ).groupby(
 [0, 1],
as_index=False ).count())
 fdmat[coord_cnt[0].astype(int),
coord_cnt[1].astype(int)] = coord_cnt['count']
return fdmat
@staticmethod
def _minimum_bounding_rectangle(hull_points):

 pi2 = np.pi / 2.
 edges = hull_points[1:] - hull_points[:-1]
angles = np.arctan2(edges[:, 1], edges[:, 0])
 angles = np.abs(np.mod(angles, pi2))
angles = np.unique(angles)
 rotations = np.vstack([
np.cos(angles),
np.cos(angles - pi2),
np.cos(angles + pi2),
np.cos(angles)
 ] ).T
rotations = rotations.reshape(( -1, 2, 2))
rot_points = np.dot(rotations, hull_points.T)
 min_x = np.nanmin(rot_points[:, 0], axis=1)
 max_x = np.nanmax(rot_points[:, 0], axis=1)
 min_y = np.nanmin(rot_points[:, 1], axis=1)
 max_y = np.nanmax(rot_points[:, 1], axis=1)
 areas =(max_x - min_x)*(max_y - min_y)
 best_idx = np.argmin(areas)
 x1 = max_x[best_idx]
x2 = min_x[best_idx]
y1 = max_y[best_idx]
y2 = min_y[best_idx]
rotmat = rotations[best_idx]
coords = np.zeros(( 4, 2))
coords[0] = np.dot([x1, y2], rotmat)
 coords[1] = np.dot([x2, y2], rotmat)
 coords[2] = np.dot([x2, y1], rotmat)
 coords[3] = np.dot([x1, y1], rotmat)
 return coords, rotmat<train_model>
def remove_numbers(text):
pattern = r'[^a-zA-z.,!?/:;"'\s]'
return re.sub(pattern, '', text)
 train.text = train.text.apply(remove_numbers)
 train.text = train.text.str.lower()
train.text = train.text.str.strip()
test.text = test.text.apply(remove_numbers)
 test.text = test.text.str.lower()
test.text = test.text.str.strip()
train['text']
class LogScaler:

 def __init__(self):
self._min0 = None
self._max = None

 def fit(self, X, y=None):
self._min0 = X.min(axis=0)
 X_norm = np.log(
 X +
np.repeat(np.abs(self._min0)[np.newaxis, :], X.shape[0], axis=0)+
1 ).clip(min=0, max=None)
 self._max = X_norm.max()

 def fit_transform(self, X, y=None):
self._min0 = X.min(axis=0)
 X_norm = np.log(
 X +
np.repeat(np.abs(self._min0)[np.newaxis, :], X.shape[0], axis=0)+
1 ).clip(min=0, max=None)
 self._max = X_norm.max()
return(X_norm / self._max ).clip(0, 1)

def transform(self, X, y=None):
for i in range(X.shape[1]):
X[:, i] = X[:, i].clip(min=self._min0[i], max=None)
 X_norm = np.log(
 X +
np.repeat(np.abs(self._min0)[np.newaxis, :], X.shape[0], axis=0)+
1 ).clip(min=0, max=None)
 return(X_norm / self._max ).clip(0, 1 )<categorify>
from sklearn.feature_extraction.text import TfidfVectorizer
class MoAImageSwapDataset(torch.utils.data.Dataset):
def __init__(self,
features,
labels,
transformer,
swap_prob=0.15,
swap_portion=0.1):
self.features = features
self.labels = labels
self.transformer = transformer
self.swap_prob = swap_prob
self.swap_portion = swap_portion
def __getitem__(self, index):
normalized = self.features[index, :]
normalized = self.add_swap_noise(index, normalized)
 normalized = np.expand_dims(normalized, axis=0)
 image = self.transformer.transform(normalized, empty_value=1)[0]
gene_cht = cv2.resize(image,(image_size, image_size),
interpolation=cv2.INTER_CUBIC)
 image = np.repeat(gene_cht[np.newaxis, :, :], 3, axis=0)
 return {"x": image, "y": self.labels[index, :]}
def add_swap_noise(self, index, X):
if np.random.rand() < self.swap_prob:
swap_index = np.random.randint(self.features.shape[0], size=1)[0]
swap_features = np.random.choice(
 np.array(range(3, self.features.shape[1])) ,
size=int(self.features.shape[1] * self.swap_portion),
replace=False)
 X[swap_features] = self.features[swap_index, swap_features]
return X
def __len__(self):
return self.features.shape[0]<normalization>
tfidf = TfidfVectorizer(min_df=1, max_df=0.3, ngram_range=(1, 2))
print(tfidf)
 text_tfidf = tfidf.fit_transform(train['text'])
 text_tfidf2 = tfidf.transform(test['text'])
class MoAImageDataset(torch.utils.data.Dataset):
def __init__(self, features, labels, transformer):
self.features = features
self.labels = labels
self.transformer = transformer
def __getitem__(self, index):
normalized = self.features[index, :]
normalized = np.expand_dims(normalized, axis=0)
 image = self.transformer.transform(normalized, empty_value=1)[0]
gene_cht = cv2.resize(image,(image_size, image_size),
interpolation=cv2.INTER_CUBIC)
 image = np.repeat(gene_cht[np.newaxis, :, :], 3, axis=0)
 return {"x": image, "y": self.labels[index, :]}
def __len__(self):
return self.features.shape[0]
class TestDataset(torch.utils.data.Dataset):
def __init__(self, features, labels, transformer):
self.features = features
self.labels = labels
self.transformer = transformer
def __getitem__(self, index):
normalized = self.features[index, :]
normalized = np.expand_dims(normalized, axis=0)
 image = self.transformer.transform(normalized, empty_value=1)[0]
gene_cht = cv2.resize(image,(image_size, image_size),
interpolation=cv2.INTER_CUBIC)
 image = np.repeat(gene_cht[np.newaxis, :, :], 3, axis=0)
 return {"x": image, "y": -1}
def __len__(self):
return self.features.shape[0]<normalization>
clf = LogisticRegression(C=4.0)
 scores = model_selection.cross_val_score(clf, text_tfidf, train["target"], cv=3, scoring="f1")
 scores
def initialize_weight_goog(m, n='', fix_group_fanout=True):
if isinstance(m, nn.Conv2d):
fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
if fix_group_fanout:
fan_out //= m.groups
m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))
if m.bias is not None:
m.bias.data.zero_()
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1.0)
 m.bias.data.zero_()
elif isinstance(m, nn.Linear):
fan_out = m.weight.size(0)
 fan_in = 0
if 'routing_fn' in n:
fan_in = m.weight.size(1)
 init_range = 1.0 / math.sqrt(fan_in + fan_out)
 m.weight.data.uniform_(-init_range, init_range)
 m.bias.data.zero_()
def initialize_weight_default(m, n=''):
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1.0)
 m.bias.data.zero_()
elif isinstance(m, nn.Linear):
nn.init.kaiming_uniform_(m.weight,
mode='fan_in',
nonlinearity='linear' )<choose_model_class>
clf.fit(text_tfidf, train["target"] )
class MoAEfficientNet(pl.LightningModule):
def __init__(
 self,
pretrained_model_name,
training_set=(None, None),
valid_set=(None, None),
test_set=None,
transformer=None,
num_classes=206,
in_chans=3,
drop_rate=0.,
drop_connect_rate=0.,
fc_size=512,
learning_rate=1e-3,
weight_init='goog'):
super(MoAEfficientNet, self ).__init__()
self.train_data, self.train_labels = training_set
self.valid_data, self.valid_labels = valid_set
self.test_data = test_set
self.transformer = transformer
self.backbone = getattr(geffnet, pretrained_model )(
 pretrained=True,
in_chans=in_chans,
drop_rate=drop_rate,
drop_connect_rate=drop_connect_rate,
weight_init=weight_init)
 self.backbone.classifier = nn.Sequential(
 nn.Linear(self.backbone.classifier.in_features, fc_size,
bias=True), nn.ELU() ,
nn.Linear(fc_size, num_classes, bias=True))
if self.training:
for m in self.backbone.classifier.modules() :
initialize_weight_goog(m)
 self.save_hyperparameters("pretrained_model_name", "num_classes",
"in_chans", "drop_rate", "drop_connect_rate",
"weight_init", "fc_size", "learning_rate")
 def forward(self, x):
return self.backbone(x)
 def training_step(self, batch, batch_idx):
x = batch["x"]
y = batch["y"]
x = x.float()
y = y.type_as(x)
 logits = self(x)
 loss = F.binary_cross_entropy_with_logits(logits, y, reduction="mean")
 self.log('train_loss',
loss,
on_step=True,
on_epoch=True,
prog_bar=True,
logger=True)
 return loss
def validation_step(self, batch, batch_idx):
x = batch["x"]
y = batch["y"]
x = x.float()
y = y.type_as(x)
 logits = self(x)
 val_loss = F.binary_cross_entropy_with_logits(logits,
y,
reduction="mean")
 self.log('val_loss',
val_loss,
on_step=True,
on_epoch=True,
prog_bar=True,
logger=True)
 return val_loss
def test_step(self, batch, batch_idx):
x = batch["x"]
y = batch["y"]
x = x.float()
y = y.type_as(x)
 logits = self(x)
 return {"pred_logits": logits}
def test_epoch_end(self, output_results):
all_outputs = torch.cat([out["pred_logits"] for out in output_results],
dim=0)
 print("Logits:", all_outputs)
 pred_probs = F.sigmoid(all_outputs ).detach().cpu().numpy()
print("Predictions: ", pred_probs)
 return {"pred_probs": pred_probs}
def setup(self, stage=None):
self.train_dataset = MoAImageSwapDataset(self.train_data,
self.train_labels,
self.transformer,
swap_prob=swap_prob,
swap_portion=swap_portion)
 self.val_dataset = MoAImageDataset(self.valid_data, self.valid_labels,
self.transformer)
 self.test_dataset = TestDataset(self.test_data, None, self.transformer)
 def train_dataloader(self):
train_dataloader = DataLoader(self.train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers,
pin_memory=True,
drop_last=False)
 print(f"Train iterations: {len(train_dataloader)}")
 return train_dataloader
def val_dataloader(self):
val_dataloader = DataLoader(self.val_dataset,
batch_size=infer_batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True,
drop_last=False)
 print(f"Validate iterations: {len(val_dataloader)}")
 return val_dataloader
def test_dataloader(self):
test_dataloader = DataLoader(self.test_dataset,
batch_size=infer_batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True,
drop_last=False)
 print(f"Test iterations: {len(test_dataloader)}")
 return test_dataloader
def configure_optimizers(self):
print(f"Initial Learning Rate: {self.hparams.learning_rate:.6f}")
 optimizer = optim.Adam(self.parameters() ,
lr=self.hparams.learning_rate,
weight_decay=weight_decay)
 scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer,
T_max=T_max,
eta_min=0,
last_epoch=-1)
 return [optimizer], [scheduler]<prepare_x_and_y>
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 submission["target"] = clf.predict(text_tfidf2)
 submission.to_csv("submission-bylogistic.csv", index=False )