Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer ,TfidfTransformer
def load_j_coupling_csv(file_path: str, train=True, verbose=False):
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
df = pd.read_csv(file_path, dtype=train_dtypes)
 df['molecule_index'] = df.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 if train:
cols = ['id', 'molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']
else:
cols = ['id', 'molecule_index', 'atom_index_0', 'atom_index_1', 'type']
df = df[cols]
df = add_coordinates(df, structures_df, 0)
 df = add_coordinates(df, structures_df, 1)
 add_center(df)
 if verbose:
print(df.shape)
 display(df.head())
 return df
train_df = load_j_coupling_csv(join(DATA_DIR, 'train.csv'), verbose=True)
 mol2distance_matrix = structures_df.groupby('molecule_index' ).apply(
 lambda df: distance_matrix(df[['x','y', 'z']].values, df[['x','y', 'z']].values))<concatenate>
nlp = spacy.load("en_core_web_lg" )
def get_knn_features_center(j_coupling: pd.Series,
structures=structures_df,
mol2dist=mol2distance_matrix,
k=10)-> np.array:
center = j_coupling[['x_c', 'y_c', 'z_c']].values.reshape(1, 3)
 mol_df = structures.loc[j_coupling.molecule_index]
coordinates = mol_df[['x','y', 'z']].values
center_distances = distance_matrix(center, coordinates ).ravel()
knn = np.argsort(center_distances)[:(k + 2)]
a_0, a_1 = j_coupling.atom_index_0, j_coupling.atom_index_1
knn = np.array([x for x in knn if x not in(a_0, a_1)])
 distances = center_distances[knn]
types = mol_df.iloc[knn].atom
distances = np.pad(distances,(0, k - len(distances)) , 'constant')
 types = np.pad(types,(0, k - len(types)) , 'constant')
 dist_matrix = mol2distance_matrix[j_coupling.molecule_index]
d_a0_a1 = dist_matrix[a_0, a_1]
knn_features = np.concatenate([[d_a0_a1], distances, types])
 return knn_features
id2center_knn = {row.id : get_knn_features_center(row)for _, row in train_df.iterrows() }<split>
train_data = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test_data =pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train_data.head(5 )
def make_data(df: pd.DataFrame, id2features: dict, random_state=128, split=True):
tmp_df = df.copy()
tmp_df['features'] = tmp_df.id.map(id2features)
 X = np.stack(tmp_df.features)
 y = tmp_df.scalar_coupling_constant.values
if split:
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=random_state)
 return(X_train, y_train),(X_val, y_val)
 else:
return X, y
LGB_PARAMS = {
'objective': 'regression',
'metric': 'mae',
'verbosity': -1,
'boosting_type': 'gbdt',
'learning_rate': 0.2,
'num_leaves': 128,
'min_child_samples': 79,
'max_depth': 9,
'subsample_freq': 1,
'subsample': 0.9,
'bagging_seed': 11,
'reg_alpha': 0.1,
'reg_lambda': 0.3,
'colsample_bytree': 1.0
}
def train_model(train, validation):
X_train, y_train = train
X_val, y_val = validation
model = LGBMRegressor(**LGB_PARAMS, n_estimators=1500, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_val, y_val)], eval_metric='mae',
verbose=100, early_stopping_rounds=200)
 y_pred = model.predict(X_val)
 score = np.log(mean_absolute_error(y_val, y_pred))
residuals = y_val - y_pred
print(f'competition-metric score: {score}')
 return model, score, residuals
def plot_residuals(residuals: np.array):
plt.hist(residuals, bins=50)
 plt.title('residual distribution')
 plt.show() ;<compute_test_metric>
train_data_shape = train_data.shape[0]
print(f'competition-metric: {np.mean(list(scores.values())) :.2f}')
 print('scores per type:')
 pprint(scores, width=1 )<prepare_x_and_y>
def clean_text(text):
url = re.compile(r'https?://\S+|www\.\S+')
 text = url.sub(r'', text)
 html = re.compile(r'<.*?>')
 text = html.sub(r'', text)
 emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags = re.UNICODE)
 text = emoji_pattern.sub(r'', text)
 table = str.maketrans('', '', string.punctuation)
 text = text.translate(table)
 text = text.lower()
text = re.sub(r"i'm", "i am", text)
 text = re.sub(r"you'll", "you will", text)
 text = re.sub(r"i'll", "i will", text)
 text = re.sub(r"she'll", "she will", text)
 text = re.sub(r"he'll", "he will", text)
 text = re.sub(r"he's", "he is", text)
 text = re.sub(r"she's", "she is", text)
 text = re.sub(r"that's", "that is", text)
 text = re.sub(r"what's", "what is", text)
 text = re.sub(r"where's", "where is", text)
 text = re.sub(r"there's", "there is", text)
 text = re.sub(r"here's", "here is", text)
 text = re.sub(r"who's", "who is", text)
 text = re.sub(r"how's", "how is", text)
 text = re.sub(r"'ll", " will", text)
 text = re.sub(r"'ve", " have", text)
 text = re.sub(r"'re", " are", text)
 text = re.sub(r"'d", " would", text)
 text = re.sub(r"can't", "cannot", text)
 text = re.sub(r"won't", "will not", text)
 text = re.sub(r"don't", "do not", text)
 text = re.sub(r"shouldn't", "should not", text)
 tweet = re.sub(r"&gt;", ">", text)
 tweet = re.sub(r"&lt;", "<", text)
 tweet = re.sub(r"&amp;", "&", text)
 tweet = re.sub(r"w/e", "whatever", text)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", text)
 tweet = re.sub(r"recentlu", "recently", text)
 tweet = re.sub(r"Ph0tos", "Photos", text)
 tweet = re.sub(r"amirite", "am I right", text)
 tweet = re.sub(r"exp0sed", "exposed", text)
 tweet = re.sub(r"<3", "love", text)
 tweet = re.sub(r"amageddon", "armageddon", text)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"8/5/2015", "2015-08-05", text)
 tweet = re.sub(r"WindStorm", "Wind Storm", text)
 tweet = re.sub(r"8/6/2015", "2015-08-06", text)
 tweet = re.sub(r"10:38PM", "10:38 PM", text)
 tweet = re.sub(r"10:30pm", "10:30 PM", text)
 tweet = re.sub(r"16yr", "16 year", text)
 tweet = re.sub(r"lmao", "laughing my ass off", text)
 tweet = re.sub(r"TRAUMATISED", "traumatized", text)
 text = re.sub(r"n't", " not", text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[{}]'.format(re.escape(string.punctuation)) , '', text)
 text = re.sub('\w*\d\w*', '', text)
 text = re.sub(r" ", " ", text)
 text = "".join([c if c not in string.punctuation else "" for c in text])
 text = ''.join(c for c in text if not c.isdigit())
 text = text.replace(r"÷", "")
 text = re.sub(' +', ' ', text)
 tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"mÌ¼sica", "music", tweet)
 tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
 tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
 tweet = re.sub(r"gawx", "Georgia Weather", tweet)
 tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
 tweet = re.sub(r"cawx", "California Weather", tweet)
 tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
 tweet = re.sub(r"azwx", "Arizona Weather", tweet)
 tweet = re.sub(r"alwx", "Alabama Weather", tweet)
 tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
 tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
 tweet = re.sub(r"Suruc", "Sanliurfa", tweet)
 return text
train_data['text'] = train_data['text'].apply(lambda x : clean_text(x))
test_data['text'] = test_data['text'].apply(lambda x : clean_text(x))
def make_test_data(df: pd.DataFrame, id2features: dict, random_state=128):
tmp_df = df.copy()
tmp_df['features'] = tmp_df.id.map(id2features)
 X = np.stack(tmp_df.features)
 return X
test_df = load_j_coupling_csv(join(DATA_DIR, 'test.csv'), train=False, verbose=True)
 id2center_knn_test = {row.id : get_knn_features_center(row)for _, row in test_df.iterrows() }
prediction_df = pd.DataFrame()
for type_ in sorted(train_df.type.unique()):
print(f'
train_type_df = train_df.query('type == @type_')
 X_train, y_train = make_data(train_type_df, id2center_knn, split=False)
 model = LGBMRegressor(**LGB_PARAMS, n_estimators=2000, n_jobs = -1)
 model.fit(X_train, y_train, eval_metric='mae')
 test_type_df = test_df.query('type == @type_')
 X_test = make_test_data(test_type_df, id2center_knn_test)
 y_hat = model.predict(X_test)
 type_pred_df = pd.DataFrame({'id': test_type_df.id, 'scalar_coupling_constant': y_hat})
 prediction_df = pd.concat([prediction_df, type_pred_df], ignore_index=True )<save_to_csv>
def massage_text(text):
tweet = re.sub("[^a-zA-Z]", ' ', text)
 tweet = tweet.lower()
tweet = tweet.split()
lem = WordNetLemmatizer()
tweet = [lem.lemmatize(word)for word in tweet
if word not in set(stopwords.words('english')) ]
tweet = ' '.join(tweet)
 return tweet
print('--here goes nothing')
 print(text)
 print(tweet)
 train_data['text'] = train_data['text'].apply(lambda x : massage_text(x))
test_data['text'] = test_data['text'].apply(lambda x : massage_text(x))
prediction_df = prediction_df.sort_values('id')
 prediction_df.to_csv('submission.csv', index=False )<load_from_csv>
count_vectorizer = feature_extraction.text.CountVectorizer()
train_vectors = count_vectorizer.fit_transform(train_data["text"])
 test_vectors = count_vectorizer.transform(test_data["text"])
 vectorizer = TfidfVectorizer()
Train = vectorizer.fit_transform(train_data['text'])
 test = vectorizer.transform(test_data['text'])
 clf = linear_model.RidgeClassifier()
clf.fit(Train, train_data["target"])
 scores = model_selection.cross_val_score(clf, Train, train_data["target"], cv=3, scoring="f1")
 scores
test = pd.read_csv('.. /input/champs-scalar-coupling/test.csv')
 sub1 = pd.read_csv('.. /input/keras-neural-net-and-distance-features/submission.csv')
 sub2 = pd.read_csv('.. /input/keras-nn-with-multi-output/submission.csv')
 display(test.head() ,sub1.head() ,sub2.head() )<create_dataframe>
clf.fit(Train, train_data["target"] )
sub = pd.DataFrame(columns = ['id','scalar_coupling_constant'])
 mol_types1 = ['2JHH','2JHN','2JHC','3JHH', '3JHC', '3JHN']
mol_types2 = ['1JHC', '1JHN']<concatenate>
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = clf.predict(test)
 sample_submission.head()
<sort_values><EOS> 
sample_submission.to_csv("submission.csv", index=False )