Kaggle competition: Natural Language Processing with Disaster Tweets
sub["count"] = np.exp(result)
 sub.head()<save_to_csv>
train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission_df = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv' )
sub.to_csv("20_03_29sub.csv",index=False )<load_from_csv>
train_df[(train_df['target'] == 0)&(train_df['keyword'].isnull())]
zf1 = zipfile.ZipFile("/kaggle/input/word2vec-nlp-tutorial/labeledTrainData.tsv.zip")
 train = pd.read_csv(zf1.open("labeledTrainData.tsv"), sep="\t")
 zf2 = zipfile.ZipFile("/kaggle/input/word2vec-nlp-tutorial/testData.tsv.zip")
 test = pd.read_csv(zf2.open("testData.tsv"), sep="\t")
 train.drop(columns=['id'], inplace=True)
 train.drop_duplicates(keep=False,inplace=True)
<load_from_csv>
train_df[(train_df['target'] == 1)&(train_df['keyword'].isnull())]
imdb_master = pd.read_csv(".. /input/imdb-review-dataset/imdb_master.csv", encoding = "ISO-8859-1")
 imdb_master.drop(inplace=True, columns=['file','type','Unnamed: 0'])
 imdb_master.columns = ['review','sentiment']
imdb_master=imdb_master[(imdb_master.sentiment == "neg")|(imdb_master.sentiment == "pos")]
di = {"neg": int("0"), "pos": int("1")}
imdb_master['sentiment']=imdb_master['sentiment'].map(di ).astype('int64')
 imdb_master.drop_duplicates(keep=False,inplace=True)
<string_transform>
print(f"No.of unique keywords in class 0: {train_df[(train_df['target'] == 0)]['keyword'].value_counts().shape[0]}")
 print(f"No.of unique keywords in class 1: {train_df[(train_df['target'] == 1)]['keyword'].value_counts().shape[0]}")
 print(f"Total no.of unique keywords: {train_df['keyword'].value_counts().shape[0]}" )
example1 = BeautifulSoup(train["review"][0])
 letters_only = re.sub("[^a-zA-Z]",
" ",
example1.get_text())
 lower_case = letters_only.lower()
words = lower_case.split()<import_modules>
URL_PATTERN = re.compile(r"(https:\/\/\S+)|(http:\/\/\S+)|(www\.\S+)")
 HTML_TAGS_PATTERN = re.compile(r'<.*>')
 ALPHA_NUMERIC_PATTERN = re.compile(r"\w*[:,-]*(\d+[:,-]*)+\d*\w*")
 PUNCTUATION_PATTERN = re.compile(r'[^a-zA-Z ]')
 MENTIONS_PATTERN = re.compile(r'@[\w]*')
 HASH_TAGS_PATTERN = re.compile(r'
UNWANTED_WORDS_PATTERN = re.compile(r'&amp;|RT: \S+:|RT \S+:|FYI|CAD|RT |GMT|UTC|JST|\s[b-zB-Z]\s|ST|nsfw')
 STOPWORDS = set(stopwords.words('english'))
def get_punctuations(text):
return PUNCTUATION_PATTERN.findall(text)
 punctuations = train_df.apply(lambda x: get_punctuations(x['text']), axis=1)
 punctuations = punctuations[punctuations.notnull() ].explode().value_counts()
fig = px.bar(x=punctuations.index, y=punctuations.values)
 fig.update_layout(title_text='Punctuation count', xaxis_title='Punctuations', yaxis_title='Count' )
import nltk 
<string_transform>
def replace_contractions(tweet):
tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 return tweet
print(stopwords.words("english"))<define_variables>
class TweetTransformer(BaseEstimator, TransformerMixin):
def __init__(self, remove_stopwords=False):

 self.remove_stopwords_ = remove_stopwords
def replace_contractions(self, tweet):

 return replace_contractions(tweet)
 def replace_urls(self, tweet):

 return URL_PATTERN.sub('', tweet)
 def replace_unwanted_words(self, tweet):

 return UNWANTED_WORDS_PATTERN.sub(' ', tweet)
 def replace_mentions(self, tweet):

 return MENTIONS_PATTERN.sub('', tweet)
 def replace_alpha_nums(self, tweet):

 return ALPHA_NUMERIC_PATTERN.sub('', tweet)
 def remove_emoji_characters(self, tweet):

 return tweet.encode('ascii', 'ignore' ).decode('ascii')
 def replace_punctuations(self, tweet):

 return PUNCTUATION_PATTERN.sub(' ', tweet)
 def modify_empty_and_rare_utterances(self, tweet):

 if tweet.strip() == '':
tweet = 'no text'
tweet = tweet.lower()
tweet = re.sub('lo+l', 'laughing out loud', tweet)
 tweet = re.sub('coo+l', 'cool', tweet)
 tweet = re.sub('(calif|cal)', 'california', tweet)
 tweet = tweet.replace('rd', 'road')
 tweet = tweet.replace('nyc', 'new york city')
 tweet = tweet.replace('sismo', 'earthquake')
 tweet = tweet.replace('detactado', 'detected')
 tweet = re.sub(' +', ' ', tweet)
 tweet = re.sub('^ +', '', tweet)
 return tweet
def remove_stopwords(self, tweet):

 tokens = word_tokenize(tweet)
 for i, token in enumerate(tokens):
if token in STOPWORDS:
del tokens[i]
return " ".join(tokens)
 def transform_helper(self, tweet):

 tweet = self.replace_contractions(tweet)
 tweet = self.replace_urls(tweet)
 tweet = self.replace_unwanted_words(tweet)
 tweet = self.replace_mentions(tweet)
 tweet = self.replace_alpha_nums(tweet)
 tweet = self.remove_emoji_characters(tweet)
 tweet = self.replace_punctuations(tweet)
 tweet = self.modify_empty_and_rare_utterances(tweet)
 if self.remove_stopwords_:
tweet = self.remove_stopwords(tweet)
 return tweet
def fit(self, X=None):
return self
def transform(self, X):

 return X.map(lambda x: self.transform_helper(x))
words = [w for w in words if not w in stopwords.words("english")]<categorify>
class NNLMEmbeddingsTransformer(BaseEstimator, TransformerMixin):
def fit(self, X=None):
return self
def transform(self, X):
return nnlm_embed(X.values )
def review_to_words(raw_review):
review_text = BeautifulSoup(raw_review ).get_text()
letters_only = re.sub("[^a-zA-Z]", " ", review_text)
 words = letters_only.lower().split()
stops = set(stopwords.words("english"))
meaningful_words = [w for w in words if not w in stops]
return(" ".join(meaningful_words))<define_variables>
class USEEmbeddingsTransformer(BaseEstimator, TransformerMixin):
def fit(self, X=None):
return self
def transform(self, X):
return use_embed(X.values )
num_reviews = train["review"].size
print("Cleaning and parsing the training set movie reviews...
 ")
 clean_train_reviews = []
for i in range(0, num_reviews):
if(( i+1)%1000 == 0):
print("Review %d of %d
" %(i+1, num_reviews))
clean_train_reviews.append(review_to_words(train["review"][i]))<choose_model_class>
use_embeddings_pipeline = Pipeline([ 
('tweet_transformer', TweetTransformer()), 
('use_embeddings_transformer', USEEmbeddingsTransformer())
 ])
 tfidf_pipeline = Pipeline([ 
('tweet_transformer', TweetTransformer(remove_stopwords=True)) , 
('tfidf_vectorizer', TfidfVectorizer())
 ])
 data_prep_pipeline = FeatureUnion([ 
('use_embeddings', use_embeddings_pipeline), 
('tfidf', tfidf_pipeline)
 ])
 X_train = data_prep_pipeline.fit_transform(train_df['text'])
 y_train = train_df['target']
X_test = data_prep_pipeline.transform(test_df['text'] )
print("Creating the bag of words...
 ")
 vectorizer = CountVectorizer(analyzer = "word", \
tokenizer = None, \
preprocessor = None, \
stop_words = None, \
max_features = 5000)
 train_data_features = vectorizer.fit_transform(clean_train_reviews)
 train_data_features = train_data_features.toarray()<train_model>
h2o.init()
forest = RandomForestClassifier(n_estimators = 100,)
 %time forest = forest.fit(train_data_features, train["sentiment"] )<choose_model_class>
train_h2o = H2OFrame(X_train.todense())
 x = train_h2o.columns
y = 'target'
train_h2o[y] = H2OFrame(y_train.values ).asfactor()
X_test_h2o = H2OFrame(X_test.todense() )
model=forest<choose_model_class>
aml = H2OAutoML(max_models=20, max_runtime_secs=18000, seed=1)
 aml.train(x=x, y=y, training_frame=train_h2o )
xgboost = xgb.XGBRegressor(colsample_bytree=0.4,
gamma=0,
learning_rate=0.07,
max_depth=3,
min_child_weight=1.5,
n_estimators=10000,
reg_alpha=0.75,
reg_lambda=0.45,
subsample=0.6,
seed=42)
 lightgbm = lgb.LGBMClassifier() 
<train_model>
predictions = aml.predict(X_test_h2o )
<categorify><EOS> 
submission_df['target'] = predictions['predict'].as_data_frame().values
submission_df.to_csv('submissions.csv', index=False)
 FileLink('submissions.csv' )