Kaggle competition: Quora Insincere Questions Classification
age_mapping = {'Baby': 1, 'Child': 2, 'Teenager': 3, 'Student': 4, 'Young': 5, 'Adult': 6, 'Senior': 7}
train['AgeGroup'] = train['AgeGroup'].map(age_mapping)
 test['AgeGroup'] = test['AgeGroup'].map(age_mapping )<drop_column>
!pip install hyperas
train=train.drop(['Age'],axis =1)
 test =test.drop(['Age'],axis=1)
 train.head()<count_missing_values>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )
train.isnull().sum()<count_missing_values>
train_df.isna().sum()
train.isnull().sum()<split>
print(train_df.duplicated(subset=['question_text'] ).sum())
 print(test_df.duplicated(subset=['question_text'] ).sum() )
X = train.drop(['Survived', 'PassengerId'], axis=1)
 y = train["Survived"]
x_train, x_val, y_train, y_val = train_test_split(X, y)
<train_model>
train_df['qlen'] = train_df['question_text'].apply(lambda x: len(x.split()))
test_df['qlen'] = test_df['question_text'].apply(lambda x: len(x.split()))
log_model = LogisticRegression()
log_model.fit(x_train, y_train)
 y_pred = log_model.predict(x_val)
 acc_log=accuracy_score(y_pred, y_val)* 100
print(acc_log )<compute_train_metric>
glove_embedding = ".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"
file = open(glove_embedding)
 embedding_index={}
for line in tqdm(file):
split_values = line.split(" ")
 embedding_index[split_values[0]] = np.asarray(split_values[1:], dtype='float32')
 file.close()
svm_model =SVC()
svm_model.fit(x_train,y_train)
 y_pred =svm_model.predict(x_val)
 acc_svc =accuracy_score(y_pred,y_val)*100
print(acc_svc)
<train_model>
def cleanup(text):
text = "".join([char for char in text if(ord(char)>=48 and ord(char)<=57)or(ord(char)>=65 and ord(char)<=90)or(ord(char)>=97 and ord(char)<=122)or(ord(char)==32)])
 text = " ".join(word if(word in embedding_index)and(word.lower() not in embedding_index)else word.lower() for word in text.split())
 return text
train = train_df.copy()
train['question_text'] = train['question_text'].apply(cleanup )
decisiontree_model =DecisionTreeClassifier()
decisiontree_model.fit(x_train,y_train)
 y_pred =decisiontree_model.predict(x_val)
 acc_decisiontree_model=accuracy_score(y_pred, y_val)*100
print(acc_decisiontree_model )<train_model>
train['qlen_wostopwords'] = train['question_text'].apply(lambda x: len(x.split()))
randomforest_model = RandomForestClassifier()
randomforest_model.fit(x_train, y_train)
 y_pred = randomforest_model.predict(x_val)
 acc_randomforest =accuracy_score(y_pred, y_val)* 100
print(acc_randomforest )<choose_model_class>
train.groupby(by=['target'] ).mean()
knn_model = KNeighborsClassifier()
knn_model.fit(x_train, y_train)
 y_pred = knn_model.predict(x_val)
 acc_knn_model = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_knn_model )<train_model>
print('Max words in train dataset: ', train.qlen_wostopwords.max())
 maxlen=100
sgd_model = SGDClassifier()
sgd_model.fit(x_train, y_train)
 y_pred = sgd_model.predict(x_val)
 acc_sgd_model = accuracy_score(y_pred, y_val)* 100
print(acc_sgd_model )<train_model>
X_train, X_test, y_train, y_test = train_test_split(train['question_text'],train['target'], test_size=0.2, random_state=42 )
gbk_model = GradientBoostingClassifier()
gbk_model.fit(x_train, y_train)
 y_pred = gbk_model.predict(x_val)
 acc_gbk_model= accuracy_score(y_pred, y_val)* 100
print(acc_gbk_model )<create_dataframe>
tokenizer = Tokenizer(num_words=50000)
 tokenizer.fit_on_texts(list(X_train))
compare =pd.DataFrame({
'model':['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest',
'Decision Tree', 'Stochastic Gradient Descent', 'Gradient Boosting Classifier'],
'Score': [acc_svc, acc_knn_model, acc_log,
acc_randomforest, acc_decisiontree_model,
acc_sgd_model, acc_gbk_model]
} )<save_to_csv>
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test )
ids = test['PassengerId']
predictions = gbk_model.predict(test.drop('PassengerId', axis=1))
output_file = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output_file.to_csv('submission.csv', index=False )<set_options>
X_train = pad_sequences(X_train,maxlen=maxlen)
 X_test = pad_sequences(X_test,maxlen=maxlen)
 y_train = y_train.values
y_test = y_test.values
%matplotlib inline
sns.set_style('whitegrid')
 pd.options.mode.chained_assignment = None
path=('.. /input/houses-2/houses.jfif')
 display(Image.open(path))<load_from_csv>
embedding_matrix = np.zeros(( 50000,300))
for word,i in tokenizer.word_index.items() :
if i>=50000: continue
v = embedding_index.get(word)
 if v is not None: embedding_matrix[i]=v
train= pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test= pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 data = pd.concat([train,test], axis = 0 , sort = False ).reset_index(drop=True)
 data.drop('SalePrice', axis =1 , inplace=True )<feature_engineering>
def f1(y_true, y_pred):
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
train["SalePrice"] = np.log1p(train["SalePrice"])
 multi_plotting(train,'SalePrice' )<count_missing_values>
inp = Input(shape=(maxlen,))
x = Embedding(50000, 300, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 max_pool = GlobalMaxPool1D()(x)
 avg_pool = GlobalAveragePooling1D()(x)
 conc = concatenate([max_pool,avg_pool])
 x = Dense(16, activation="relu" )(conc)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])
 print(model.summary() )
def missing_rate(frame):
TL = frame.isna().sum() /len(frame)
 TCMR = TL[TL >0.6].index
frame.drop(TCMR, axis= 1 , inplace = True)
 print(f'Colums to be dropped:{TCMR}')
 missing_rate(train)
 missing_rate(data )<data_type_conversions>
model.fit(X_train,y_train,validation_data=(X_test,y_test),epochs=3,batch_size=1024 )
numeric = data.dtypes[data.dtypes != 'object'].index
tnumeric = train.dtypes[train.dtypes != 'object'].index
integer = data[numeric]
tinteger = train[tnumeric]
integer.fillna(0, inplace=True)
 tinteger.fillna(0 , inplace = True )<merge>
pred = model.predict(X_test)
Final = Feature.merge(Target,left_on='level_1', right_on='index')
 Final<drop_column>
thresh=0.1
prev_score=0
for i in np.arange(0.1,0.41,0.01):
score = f1_score(y_test,(pred>i))
if score>prev_score: thresh=i
prev_score=score
print('threshold at ', i, score)
 print('Best threshold: ', thresh, ' f1score: ', f1_score(y_test,(pred>thresh)) )
tinteger.drop(['GarageArea','GarageYrBlt','TotRmsAbvGrd','Id'],axis=1, inplace=True)
 integer.drop(['GarageArea','GarageYrBlt','TotRmsAbvGrd','Id'],axis=1, inplace=True )<feature_engineering>
test_df['question_text'] = test_df['question_text'].apply(cleanup)
 test_X = tokenizer.texts_to_sequences(test_df['question_text'].values)
 test_X = pad_sequences(test_X,maxlen=maxlen )
tinteger['Porch'] = tinteger['3SsnPorch']+tinteger['ScreenPorch']+tinteger['EnclosedPorch']+tinteger['OpenPorchSF']
tinteger['YNPorch'] = tinteger['Porch'].apply(lambda x : 1 if x>0 else 0)
 tinteger.drop(['3SsnPorch','ScreenPorch','EnclosedPorch','OpenPorchSF','PoolArea'], axis= 1, inplace=True)
 integer['Porch'] = integer['3SsnPorch']+integer['ScreenPorch']+integer['EnclosedPorch']+integer['OpenPorchSF']
integer['YNPorch'] = integer['Porch'].apply(lambda x : 1 if x>0 else 0)
 integer.drop(['3SsnPorch','ScreenPorch','EnclosedPorch','OpenPorchSF','PoolArea'], axis= 1, inplace=True)
<feature_engineering>
test_pred = model.predict(test_X)
 test_pred =(test_pred>thresh ).astype('int' )
<concatenate><EOS> 
pd.DataFrame({'qid':test_df['qid'].values, 'prediction':np.squeeze(test_pred)} ).to_csv('submission.csv',index=False)