Kaggle competition: Natural Language Processing with Disaster Tweets
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 sub = pd.read_csv('.. /input/sample_submission.csv')
 train.info()
train.shape
train.describe()<train_on_grid>
train=pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test=pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
train_feat = train.drop(columns=['id', 'target'], axis=1)
 test_feat = test.drop(columns='id', axis=1)
 std = StandardScaler()
train_feat = std.fit_transform(train_feat)
 test_feat = std.fit_transform(test_feat)
 train_label = train['target']
l1 = LassoCV().fit(train_feat, train_label)
 l2 = LogisticRegressionCV().fit(train_feat, train_label)
 pred3 = l1.predict(test_feat)
 pred4 = l2.predict(test_feat)
 target = pd.DataFrame(pred3 ).rename(columns={0: 'target'})
 sub_id = sub[['id']]
submission = pd.concat([sub_id, target], axis=1)
 print(submission)
 submission.to_csv('sub.csv', index=False )<set_options>
occurences['Percentage(%)']=(occurences['Number of Occurences']/occurences['Number of Occurences'].sum())*100
occurences.set_index('Class' )
warnings.filterwarnings('ignore')
 pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', -1)
 pd.set_option('display.expand_frame_repr', False)
 pd.options.display.max_columns = None<load_from_csv>
traindata = list(np.array(train.iloc[:,3]))
testdata = list(np.array(test.iloc[:,3]))
y = np.array(train.iloc[:,4] ).astype(int)
 X_all = traindata + testdata
lentrain = len(traindata )
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 train_label = train_data.pop('target')
 test_id = test_data.pop('id')
 train_data.drop('id', axis=1, inplace=True)
 stats_data = pd.concat([train_data, test_data])
 print('Train rows:', train_data.shape)
 print('Test rows:', test_data.shape)
 print('Stats rows:', stats_data.shape )<categorify>
tfidf = TfidfVectorizer(min_df=3, max_features=None, strip_accents='unicode',
analyzer='word',token_pattern=r'\w{1,}',ngram_range=(1, 2), use_idf=1,smooth_idf=1,sublinear_tf=1)
 print("Implementing TFIDF to both the test and train data")
 tfidf.fit(X_all)
 print("Transforming the data")
 X_all = tfidf.transform(X_all )
def normDist_WoE(frame, target, nrOfBins):
ranges = []
for i in range(0, nrOfBins + 1):
ranges.append(norm.ppf(i / nrOfBins, loc=0, scale=1))
df = pd.DataFrame(frame)
 df.columns = ['val']
df['target'] = target.astype(int)
 df['bin'] = 0
for i in range(1, nrOfBins + 1):
df.loc[(df['val'] >= ranges[i-1])&(df['val'] < ranges[i]), 'bin'] = i
result = []
for i in range(1, nrOfBins + 1):
dic = {}
dic.update(BinNr=i, Min_Value=ranges[i-1], Max_Value=ranges[i], Count=len(df[df['bin'] == i]),
Event=len(df[(df['bin'] == i)&(df['target'] == 1)]),
NonEvent=len(df[(df['bin'] == i)&(df['target'] != 1)]))
result.append(dic)
 result = pd.DataFrame(result)
 result = result.reindex(columns=(['BinNr', 'Min_Value', 'Max_Value', 'Count', 'Event', 'NonEvent']))
result['Event_Rate'] = result['Event'] / result['Count']
result['NonEvent_Rate'] = result['NonEvent'] / result['Count']
result['Dist_Event'] = result['Event'] / sum(result['Event'])
 result['Dist_NonEvent'] = result['NonEvent'] / sum(result['NonEvent'])
 result['WoE'] = np.log(result['Dist_Event'] / result['Dist_NonEvent'])
 result['IV'] =(result['Dist_Event'] - result['Dist_NonEvent'])* np.log(result['Dist_Event'] /result['Dist_NonEvent'])
 return(sum(result['IV']), result )<create_dataframe>
X = X_all[:lentrain]
X_test = X_all[lentrain:]
log = LogisticRegression(penalty='l2',dual=False, tol=0.0001,
C=1, fit_intercept=True, intercept_scaling=1.0,
class_weight=None, random_state=None)
 print("Training on the train data")
 log.fit(X,y)
 y_pred_X=log.predict(X)
 print('Training accuracy is {}'.format(accuracy_score(y, y_pred_X)))
 predictions = log.predict(X_test)
 predictions
<categorify><EOS> 
test_ids=test['id']
submission = pd.DataFrame(predictions,index=test_ids,columns=['target'])
 submission.to_csv('submission_nlprnot.csv')
 print("submission file created.. " )