Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<train_model>
plt.rcParams.update({'font.size': 14})
 train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 sub_sample = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 print(train.shape, test.shape, sub_sample.shape )
end_lda=time.time()
print("Time till Lda model viz:",end_lda-start_time,"s" )<drop_column>
train.duplicated().sum()
del(ldamodel)
 del(corpus)
 del(dictionary)
 del(model)
 gc.collect()<categorify>
train = train.drop_duplicates().reset_index(drop=True )
def preprocess_and_clean(text_col):

 word_list = gensim.utils.simple_preprocess(text_col, deacc=True)
 clean_words = [w for w in word_list if not w in eng_stopwords]
clean_words=[lem.lemmatize(word, "v")for word in clean_words]
return(' '.join(clean_words))
print("Before clean:",train.project_description.iloc[16])
 print("After clean:",preprocess_and_clean(train.project_description.iloc[16]))<prepare_x_and_y>
train.isnull().sum()
train.teacher_prefix=train.teacher_prefix.fillna('Unknown')
 test.teacher_prefix=test.teacher_prefix.fillna('Unknown')
 train.gender=train.gender.fillna('Unknown')
 test.gender=test.gender.fillna('Unknown')
 print(train.shape)
 print(test.shape)
 y=train['project_is_approved']
train_id=train['id']
test_id=test['id']
del(train['project_is_approved'])
 all_data=pd.concat([train,test],axis=0)
 print(all_data.shape)
 all_data_merge=pd.merge(left=all_data,right=resources_agg,on='id',how='left' )<import_modules>
test.isnull().sum()
<categorify>
test.isnull().sum()
cols = [
'teacher_id',
'teacher_prefix',
'school_state',
'project_grade_category',
'project_subject_categories',
'project_subject_subcategories',
'gender']
for c in tqdm(cols):
le = preprocessing.LabelEncoder()
le.fit(all_data_merge[c].astype(str))
all_data_merge[c] = le.transform(all_data_merge[c].astype(str))<feature_engineering>
print(train.keyword.nunique() , test.keyword.nunique())
 print(set(train.keyword.unique())- set(test.keyword.unique()))
all_data_merge['avg_price_per_item']=np.log1p(all_data_merge['avg_price_per_item'])
 all_data_merge['total_cost']=np.log1p(all_data_merge['total_cost'])
 all_data_merge['project_submitted_datetime'] = pd.to_datetime(train['project_submitted_datetime'])
 all_data_merge['datetime_dow'] = all_data_merge['project_submitted_datetime'].dt.dayofweek
all_data_merge['datetime_year'] = all_data_merge['project_submitted_datetime'].dt.year
all_data_merge['datetime_month'] = all_data_merge['project_submitted_datetime'].dt.month
all_data_merge['datetime_hour'] = all_data_merge['project_submitted_datetime'].dt.hour
all_data_merge['datetime_day'] = all_data_merge['project_submitted_datetime'].dt.day<feature_engineering>
print(train.location.nunique() , test.location.nunique() )
text_cols=['project_title',
'collated_description',
'project_resource_summary',
'student_description',
'project_description']
for c in tqdm(text_cols):
all_data_merge[c+'_len']=all_data_merge[c].apply(len)
 all_data_merge[c+'_word_count']=all_data_merge[c].apply(lambda x: len(str(x ).split()))
all_data_merge[c]=all_data_merge[c].apply(preprocess_and_clean)
 end_time=time.time()
print("Time till end",end_time-start_time,"s" )<drop_column>
for col in ['keyword','location']:
train[col] = train[col].fillna('None')
 test[col] = test[col].fillna('None')
 def clean_loc(x):
if x == 'None':
return 'None'
elif x == 'Earth' or x =='Worldwide' or x == 'Everywhere':
return 'World'
elif 'New York' in x or 'NYC' in x:
return 'New York'
elif 'London' in x:
return 'London'
elif 'Mumbai' in x:
return 'Mumbai'
elif 'Washington' in x and 'D' in x and 'C' in x:
return 'Washington DC'
elif 'San Francisco' in x:
return 'San Francisco'
elif 'Los Angeles' in x:
return 'Los Angeles'
elif 'Seattle' in x:
return 'Seattle'
elif 'Chicago' in x:
return 'Chicago'
elif 'Toronto' in x:
return 'Toronto'
elif 'Sacramento' in x:
return 'Sacramento'
elif 'Atlanta' in x:
return 'Atlanta'
elif 'California' in x:
return 'California'
elif 'Florida' in x:
return 'Florida'
elif 'Texas' in x:
return 'Texas'
elif 'United States' in x or 'USA' in x:
return 'USA'
elif 'United Kingdom' in x or 'UK' in x or 'Britain' in x:
return 'UK'
elif 'Canada' in x:
return 'Canada'
elif 'India' in x:
return 'India'
elif 'Kenya' in x:
return 'Kenya'
elif 'Nigeria' in x:
return 'Nigeria'
elif 'Australia' in x:
return 'Australia'
elif 'Indonesia' in x:
return 'Indonesia'
elif x in top_loc:
return x
else: return 'Others'
train['location_clean'] = train['location'].apply(lambda x: clean_loc(str(x)))
 test['location_clean'] = test['location'].apply(lambda x: clean_loc(str(x)) )
train_shape=train.shape
test_shape=test.shape
del(train_merge)
 del(resources)
 del(resources_agg)
 del(subset)
 del(approvals)
 del(rejects)
 del(all_data)
 gc.collect()<train_model>
leak = pd.read_csv(".. /input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv")
 leak['target'] =(leak['choose_one']=='Relevant' ).astype(int)
 leak['id'] = leak.index
leak = leak[['id', 'target','text']]
merged_df = pd.merge(test, leak, on='id')
 sub1 = merged_df[['id', 'target']]
sub1.to_csv('submit_1.csv', index=False )
<categorify>
test_str = train.loc[417, 'text']
def clean_text(text):
text = re.sub(r'https?://\S+', '', text)
 text = re.sub(r'
',' ', text)
 text = re.sub('\s+', ' ', text ).strip()
return text
print("Original text: " + test_str)
 print("Cleaned text: " + clean_text(test_str))
del(all_data_merge['project_submitted_datetime'])
 all_data_merge_1=all_data_merge.to_dict('records')
 del(all_data_merge)
 gc.collect()<import_modules>
def find_hashtags(tweet):
return " ".join([match.group(0)[1:] for match in re.finditer(r"
def find_mentions(tweet):
return " ".join([match.group(0)[1:] for match in re.finditer(r"@\w+", tweet)])or 'no'
def find_links(tweet):
return " ".join([match.group(0)[1:] for match in re.finditer(r"https?://\S+", tweet)])or 'no'
def process_text(df):
df['text_clean'] = df['text'].apply(lambda x: clean_text(x))
df['hashtags'] = df['text'].apply(lambda x: find_hashtags(x))
df['mentions'] = df['text'].apply(lambda x: find_mentions(x))
df['links'] = df['text'].apply(lambda x: find_links(x))
return df
train = process_text(train)
 test = process_text(test )
def get_col(col_name):
return lambda x: x[col_name]
vectorizer = FeatureUnion([ 
('project_title',CountVectorizer(
 ngram_range=(1, 2),
max_features=10000,
preprocessor=get_col('project_title'))), 
('project_resource_summary',TfidfVectorizer(
 ngram_range=(1, 2),
max_features=30000,
preprocessor=get_col('project_resource_summary'))), 
('student_description',TfidfVectorizer(
 ngram_range=(1, 2),
max_features=100000,
preprocessor=get_col('student_description'))), 
('project_description',TfidfVectorizer(
 ngram_range=(1, 2),
max_features=100000,
preprocessor=get_col('project_description'))), 
('collated_description',CountVectorizer(
 ngram_range=(1, 2),
max_features=30000,
preprocessor=get_col('collated_description'))), 
('Non_text',DictVectorizer())
 ] )<feature_engineering>
def create_stat(df):
df['text_len'] = df['text_clean'].apply(len)
 df['word_count'] = df["text_clean"].apply(lambda x: len(str(x ).split()))
df['stop_word_count'] = df['text_clean'].apply(lambda x: len([w for w in str(x ).lower().split() if w in STOPWORDS]))
df['punctuation_count'] = df['text_clean'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
df['hashtag_count'] = df['hashtags'].apply(lambda x: len(str(x ).split()))
df['mention_count'] = df['mentions'].apply(lambda x: len(str(x ).split()))
df['link_count'] = df['links'].apply(lambda x: len(str(x ).split()))
df['caps_count'] = df['text_clean'].apply(lambda x: sum(1 for c in str(x)if c.isupper()))
df['caps_ratio'] = df['caps_count'] / df['text_len']
return df
train = create_stat(train)
 test = create_stat(test)
 print(train.shape, test.shape )
start_vect=time.time()
all_data_vectorized = vectorizer.fit_transform(all_data_merge_1)
 end_time=time.time()
print("total time in vectorization creation",end_time-start_vect,"s")
 print("total time till vectorization creation",end_time-start_time,"s" )<create_dataframe>
train.corr() ['target'].drop('target' ).sort_values()
final_dataset=all_data_vectorized.tocsr()<prepare_x_and_y>
stopwords = set(STOPWORDS)
train_x=final_dataset[0:train_shape[0]]
test_x=final_dataset[train_shape[0]:]<set_options>
features = ['keyword', 'location_clean']
encoder = ce.TargetEncoder(cols=features)
 encoder.fit(train[features],train['target'])
 train = train.join(encoder.transform(train[features] ).add_suffix('_target'))
test = test.join(encoder.transform(test[features] ).add_suffix('_target'))
del final_dataset,all_data_vectorized
gc.collect()<split>
features_to_drop = ['id', 'keyword','location','text','location_clean','text_clean', 'hashtags', 'mentions','links']
X_train = train.drop(columns = features_to_drop + ['target'])
 X_test = test.drop(columns = features_to_drop)
 y_train = train.target
lr = LogisticRegression(solver = 'liblinear')
 lr.fit(X_train, y_train)
 cv5 = cross_val_score(lr, X_train, y_train, cv=5, scoring='f1')
 print(cv5)
 print(np.mean(cv5))
X_train, X_valid, y_train, y_valid = train_test_split(train_x, y, test_size=0.33, random_state=2018)
 params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'max_depth': 16,
'num_leaves': 31,
'learning_rate': 0.25,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'verbose': 1,
'num_threads': 4,
'lambda_l2': 1,
'min_gain_to_split': 0,
'seed':1234
}<train_model>
y_test = lr.predict(X_test)
 submit = sub_sample.copy()
submit.target = y_test
submit.to_csv('submit_lr.csv',index=False )
model = lgb.train(
 params,
lgb.Dataset(X_train, y_train),
num_boost_round=10000,
valid_sets=[lgb.Dataset(X_valid, y_valid)],
early_stopping_rounds=100,
verbose_eval=25 )<compute_train_metric>
submit_lr = pd.read_csv(".. /input/socialmediadisastertweetsdfecsv/submit_lr.csv" )
<train_model><EOS> 
socialmedia_disaster_tweets-DFE = pd.read_csv(".. /input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv" )