Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<find_best_params>
stop = set(stopwords.words('english'))
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
amw.model_full_path<predict_on_test>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 target = train['target']
print("Train shape", train.shape)
 print("Test shape", test.shape )
print(f'Begin getting predictions at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 prediction_client = automl.PredictionServiceClient()
amw.set_prediction_client(prediction_client)
 predictions_df = amw.get_predictions(test,
input_col_name='text',
limit=None,
threshold=0.5,
verbose=False)
 print(f'Finished getting predictions at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}' )<concatenate>
def create_corpus(target):

 corpus = []
for x in train[train['target'] == target]['text'].str.split() :
for i in x:
corpus.append(i)
 return corpus
def filter_specific_word(corpus, filters):

 dic = defaultdict(int)
 for word in corpus:
if word in filters:
dic[word] += 1
return dic
submission_df = pd.concat([test['id'], predictions_df['class']], axis=1 )<rename_columns>
def get_top_tweet_bigrams(corpus, n=None):
vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in vec.vocabulary_.items() ]
words_freq = sorted(words_freq, key=lambda x: x[1], reverse=True)
 return words_freq
submission_df = submission_df.rename(columns={'class':'target'})
 submission_df.head()<save_to_csv>
df = pd.concat([train, test], axis=0, sort=False)
 df.shape
submission_df.to_csv("submission.csv", index=False, header=True )<import_modules>
def remove_url(text):
url = re.compile('https?://\S+|www\.\S+')
 return url.sub(r'', text)
 remove_url('http://www.kaggle.com/rakkaalhazimi/nlp-disaster-classification/edit?rvi=1' )
import transformers
import torch.nn as nn
import torch
from tqdm import tqdm
import torch
import torch.nn as nn
import pandas as pd
import torch.nn as nn
import numpy as np
from sklearn import model_selection
from sklearn import metrics
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup<load_pretrained>
df['text'] = df['text'].apply(lambda x: remove_url(x))
retain = df['text'].str.contains(r'http[s]*' ).sum()
print("{} words were left behind".format(retain))
MAX_Len = 512
TRAIN_BATCH_SIZE =8
VALID_BATCH_SIZE = 4
BERT_PATH = '.. /input/bert-base-uncased'
TOKENZIER = transformers.BertTokenizer.from_pretrained(BERT_PATH ,do_lower_case = True )<normalization>
residual = df[df['text'].str.contains(r'http[s]*')]
left_word = []
for i in range(len(residual)) :
print(residual['text'].values[i])
 left_word.append(residual['text'].values[i] )
class BertBaseUncased(nn.Module):
def __init__(self):
super(BertBaseUncased,self ).__init__()
self.bert = transformers.BertModel.from_pretrained(BERT_PATH)
 self.bert_drop = nn.Dropout(0.4)
 self.out = nn.Linear(768,1)
 def forward(self,ids,mask,token_type_ids):
out1,out2 = self.bert(
 ids ,
attention_mask = mask ,
token_type_ids = token_type_ids 
)
 bo = self.bert_drop(out2)
 output = self.out(bo)
 return output<categorify>
for word in left_word:
compiler = re.compile(r'.http.+')
 result = compiler.sub('', word)
 print(result )
class BERTDataset :
def __init__(self,df):
self.text = df['text'].values
self.target = df['target'].values
self.tokenizer = TOKENZIER
self.max_len = MAX_Len
def __len__(self):
return len(self.text)
 def __getitem__(self, item):
text = str(self.text[item])
 text = " ".join(text.split())
 inputs = self.tokenizer.encode_plus(
 text,
None,
add_special_tokens=True,
max_length=self.max_len 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
token_type_ids = inputs["token_type_ids"]
padding_length = self.max_len - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),
'targets': torch.tensor(self.target[item], dtype=torch.float)
 }<compute_test_metric>
df['text'] = df['text'].str.replace(r'.http.+', '')
 print("http words found {}".format(df['text'].str.contains('http' ).sum()))
def loss_fn(outputs, targets):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))<train_model>
def remove_html(text):
html = re.compile(r'<.*?>')
 return html.sub(r'', text)
 print(remove_html(example))
def train_fn(data_loader, model, optimizer, scheduler):
model.train()
for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)) :
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step() 
<categorify>
df['text'] = df['text'].apply(lambda x: remove_html(x))
def eval_fn(data_loader, model):
model.eval()
fin_targets = []
fin_outputs = []
with torch.no_grad() :
for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)) :
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 fin_targets.extend(targets.cpu().detach().numpy().tolist())
 fin_outputs.extend(torch.sigmoid(outputs ).cpu().detach().numpy().tolist())
 return fin_outputs, fin_targets<load_from_csv>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 remove_emoji("Omg another Earthquake 😔😔" )
DEVICE =torch.device("cuda")
 device = torch.device("cuda")
 def run(model,EPOCHS):
dfx = pd.read_csv('.. /input/nlp-getting-started/train.csv' ).fillna("none")
 df_train, df_valid = model_selection.train_test_split(
 dfx,
test_size=0.1,
random_state=42,
stratify=dfx.target.values 
)
 train_dataset = BERTDataset(
 df_train 
)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
num_workers=4 
)
 valid_dataset = BERTDataset(
 df_valid 
)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=VALID_BATCH_SIZE,
num_workers=1 
)
 device = torch.device("cuda")
 param_optimizer = list(model.named_parameters())
 no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},
]
num_train_steps = int(len(train_data_loader)) * EPOCHS
optimizer = AdamW(optimizer_parameters, lr=1e-5)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 model = nn.DataParallel(model)
 best_accuracy = 0
for epoch in range(EPOCHS):
train_fn(train_data_loader, model, optimizer, scheduler)
 outputs, targets = eval_fn(valid_data_loader, model)
 outputs = np.array(outputs)>= 0.5
accuracy = metrics.accuracy_score(targets, outputs)
 print(f"Accuracy Score = {accuracy}")
 scheduler.step() 
<string_transform>
df['text'] = df['text'].apply(lambda x: remove_emoji(x))
def sentence_prediction(sentence):
tokenizer = TOKENZIER
max_len = MAX_Len
text = str(sentence)
 text = " ".join(text.split())
 inputs = tokenizer.encode_plus(
 text,
None,
add_special_tokens=True,
max_length=max_len 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
token_type_ids = inputs["token_type_ids"]
padding_length = max_len - len(ids)
 ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 ids = torch.tensor(ids, dtype=torch.long ).unsqueeze(0)
 mask = torch.tensor(mask, dtype=torch.long ).unsqueeze(0)
 token_type_ids = torch.tensor(token_type_ids, dtype=torch.long ).unsqueeze(0)
 ids = ids.to(DEVICE, dtype=torch.long)
 token_type_ids = token_type_ids.to(DEVICE, dtype=torch.long)
 mask = mask.to(DEVICE, dtype=torch.long)
 outputs = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs = torch.sigmoid(outputs ).cpu().detach().numpy()
return outputs[0][0]<load_from_csv>
def remove_punct(text):
table = str.maketrans('', '', string.punctuation)
 return text.translate(table)
 example = "I am King
remove_punct(example )
test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 test['target'] = test['text'].apply(sentence_prediction )<prepare_output>
df['text'] = df['text'].apply(lambda x: remove_punct(x))
sub = test[['id','target']]<data_type_conversions>
df = pd.read_csv(".. /input/nlp-disaster-cleaned/tweetDisaster.csv" )
sub['target'] = sub['target'].round().astype('int' )<load_from_csv>
def create_corpus(df):
copy_df = df.copy()
corpus = []
for tweet in tqdm(copy_df["text"]):
words = [word.lower() for word in word_tokenize(tweet)if(( word.isalpha() == 1)&(word not in stop)) ]
corpus.append(words)
 return corpus
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 train = train.fillna('None')
 ag = train.groupby('keyword' ).agg({'text':np.size, 'target':np.mean} ).rename(columns={'text':'Count', 'target':'Disaster Probability'})
 ag.sort_values('Disaster Probability', ascending=False ).head(10 )<filter>
corpus = create_corpus(df )
keyword_list = list(ag[(ag['Count']>2)&(ag['Disaster Probability']>=0.9)].index)
 keyword_list<feature_engineering>
embedding_dict = {}
with open("/kaggle/input/glove-global-vectors-for-word-representation/glove.6B.100d.txt")as f:
for line in f:
values = line.split()
word = values[0]
vectors = np.asarray(values[1:], 'float32')
 embedding_dict[word] = vectors
ids = test['id'][test.keyword.isin(keyword_list)].values
sub['target'][sub['id'].isin(ids)] = 1
sub.head()<save_to_csv>
print("Embedding shape :({},{})".format(len(embedding_dict),
len(embedding_dict['the'])) )
sub.to_csv('submission.csv',index=False )<import_modules>
MAX_LEN = 50
tokenizer_obj = Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences = tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad = pad_sequences(sequences, maxlen=MAX_LEN, truncating='post', padding='post' )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt<load_from_csv>
word_index = tokenizer_obj.word_index
print("Number of unique words:", len(word_index))
features = pd.read_csv('.. /input/covid19-global-forecasting-week-1/train.csv')
 test_features = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv' )<count_missing_values>
top = sorted(word_index, key=lambda x: word_index[x], reverse=True)[:10]
unknown_index = []
for word in top:
scores =(word, word_index[word])
 unknown_index.append(scores)
 unknown_index
features.isnull().sum()<count_missing_values>
num_words = len(word_index)+ 1
embedding_matrix = np.zeros(( num_words, 100))
for word, i in tqdm(word_index.items()):
if i > num_words:
continue
emb_vec = embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i] = emb_vec
test_features.isnull().sum()<data_type_conversions>
train = tweet_pad[:train.shape[0]]
test = tweet_pad[train.shape[0]:]
features["Date"] = features["Date"].apply(lambda x: x.replace("-",""))
features["Date"] = features["Date"].astype(int)
 features.head()<data_type_conversions>
model = Sequential()
embedding = Embedding(num_words, 100, embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN, trainable=False)
 model.add(embedding)
 model.add(Bidirectional(LSTM(128,
dropout=0.2,
recurrent_dropout=0.2)) 
)
 model.add(Dense(1, activation='sigmoid'))
optimizer = Adam(learning_rate=1e-4)
 model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
 model.summary()
test_features["Date"] = test_features["Date"].apply(lambda x: x.replace("-",""))
test_features["Date"] = test_features["Date"].astype(int)
 test_features.head()<prepare_x_and_y>
history = model.fit(train, target, batch_size=32, epochs=15,
validation_split=0.2, verbose=1 )
train1_x = features[['Lat','Long','Date']]
train1_y = features[['ConfirmedCases']]
test1_x = test_features[['Lat','Long','Date']]<train_model>
sample_sub = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_sub.shape
model = RandomForestClassifier(max_depth=200, random_state=1)
 model.fit(train1_x, train1_y)
 conf_cases = model.predict(test1_x )<prepare_x_and_y>
y_pred = model.predict(test)
 y_pred = np.round(y_pred ).astype(int ).reshape(3263)
 sub = pd.DataFrame({'id': sample_sub['id'].values.tolist() , 'target': y_pred})
 sub.to_csv("submission.csv", index=False )