Kaggle competition: Fake News e ML
<SOS> metric: categorizationaccuracy Kaggle data source: fake-news-e-ml<feature_engineering>
%%capture
train = pd.read_csv('.. /input/fakenewsvortexbsb/train_df.csv', sep=';', error_bad_lines=False, quoting=3);
df_all['LotFrontage'] = df_all['LotFrontage'].apply(LotFrontage )<categorify>
example = train["manchete"][1]
print(unidecode(example))
df_dummy = pd.get_dummies(df_all['Neighborhood'],prefix='Neighborhood',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Neighborhood']<categorify>
letters_only=re.sub("[^a-zA-Z]"," ",unidecode(example))
print(letters_only )
df_all['Exterior2nd'] = df_all['Exterior2nd'].fillna('Other')
 df_dummy = pd.get_dummies(df_all['Exterior2nd'],prefix='Exterior2nd',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Exterior2nd']<categorify>
lower_case=letters_only.lower()
words=lower_case.split()
df_dummy = pd.get_dummies(df_all['Utilities'],prefix='Utilities',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Utilities']<categorify>
print(stopwords.words("portuguese"))
df_dummy = pd.get_dummies(df_all['LandContour'],prefix='LandContour',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['LandContour']<categorify>
stop = stopwords.words("portuguese" )
df_dummy = pd.get_dummies(df_all['Condition1'],prefix='Condition1',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Condition1']<categorify>
lista_stop = [unidecode(x)for x in stop]
print(lista_stop )
def Condition2(c):
if c == 'Norm':
return 1
elif c == 'Feedr':
return 2
elif c == 'PosA':
return 3
elif c == 'PosN':
return 4
elif c == 'Artery':
return 5
else:
return 0<data_type_conversions>
words=[w for w in words if not w in lista_stop]
print(words )
df_all['Condition2'] = df_all['Condition2'].apply(Condition2 ).astype(int )<categorify>
def review_to_words(raw_review):
raw_review = unidecode(raw_review)
 raw_review.lstrip('Jovem Pan')
 letters_only=re.sub("[^a-zA-Z]"," ",raw_review)
 words=letters_only.lower().split()
meaningful_words=[w for w in words if not w in lista_stop]
return(' '.join(meaningful_words))
df_dummy = pd.get_dummies(df_all['BldgType'],prefix='BldgType',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BldgType']<categorify>
clean_review=review_to_words(train['manchete'][1])
 print(clean_review )
df_dummy = pd.get_dummies(df_all['HouseStyle'],prefix='HouseStyle',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['HouseStyle']<categorify>
num_reviews=train['manchete'].size
print(num_reviews )
df_dummy = pd.get_dummies(df_all['RoofMatl'],prefix='RoofMatl',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['RoofMatl']<data_type_conversions>
clean_train_review=[]
for i in range(0,num_reviews):
clean_train_review.append(review_to_words(train['manchete'][i]))
df_all['Exterior1st']= df_all['Exterior1st'].fillna('Other' )<categorify>
vectorizer=CountVectorizer(analyzer='word',tokenizer=None,preprocessor = None, stop_words = None,max_features = 7000)
 train_data_features=vectorizer.fit_transform(clean_train_review)
 train_data_features=train_data_features.toarray()
def Exterior1st(c):
if c == 'VinylSd':
return 1
elif c == 'Wd Sdng':
return 2
elif c == 'HdBoard':
return 3
elif c == 'Plywood':
return 4
elif c == 'MetalSd':
return 5
elif c == 'CemntBd':
return 6
elif c == 'WdShing':
return 7
elif c == 'BrkFace':
return 8
elif c == 'AsbShng':
return 9
else:
return 0<data_type_conversions>
vcab=vectorizer.get_feature_names()
df_all['Exterior1st'] = df_all['Exterior1st'].apply(Exterior1st ).astype(int )<categorify>
train_y = train["Class"]
def BsmtFinSF1(n):
if(n > -1)&(n <= 150):
return 1
elif(n > 150)&(n <= 300):
return 2
elif(n > 300)&(n <= 450):
return 3
elif(n > 450)&(n <= 600):
return 4
elif(n > 600)&(n <= 750):
return 5
else:
return 10<data_type_conversions>
X_train, X_test, y_train, y_test = train_test_split(train_data_features, train_y, test_size=0.25, random_state=42 )
df_all['BsmtFinSF1'] = df_all['BsmtFinSF1'].apply(BsmtFinSF1 ).astype(int )<concatenate>
model = KNeighborsClassifier(n_neighbors=3)
 %time model = model.fit(X_train, y_train )
def MasVnrArea(n):
if(n > -1)&(n <= 150):
return 1
elif(n > 150)&(n <= 200):
return 2
elif(n > 200)&(n <= 300):
return 3
else:
return 0<data_type_conversions>
result = model.predict(X_test )
df_all['MasVnrArea'] = df_all['MasVnrArea'].apply(MasVnrArea ).astype(int )<categorify>
accuracy_score(y_test, result )
df_dummy = pd.get_dummies(df_all['ExterQual'],prefix='ExterQual',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['ExterQual']<categorify>
print(classification_report(y_test, result))
df_dummy = pd.get_dummies(df_all['ExterCond'],prefix='ExterCond',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['ExterCond']<categorify>
nb = MultinomialNB()
%time nb = nb.fit(X_train, y_train)
 result = nb.predict(X_test )
df_all['BsmtQual'] = df_all['BsmtQual'].fillna('TA')
 df_dummy = pd.get_dummies(df_all['BsmtQual'],prefix='BsmtQual',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BsmtQual']<categorify>
accuracy_score(y_test, result )
df_all['BsmtCond'] = df_all['BsmtCond'].fillna('TA')
 df_dummy = pd.get_dummies(df_all['BsmtCond'],prefix='BsmtCond',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BsmtCond']<categorify>
print(classification_report(y_test, result))
df_all['BsmtExposure'] = df_all['BsmtExposure'].fillna('TA')
 df_dummy = pd.get_dummies(df_all['BsmtExposure'],prefix='BsmtExposure',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BsmtExposure']<categorify>
clf2 = DecisionTreeClassifier(random_state=42)
 %time clf2 = clf2.fit(X_train, y_train)
 result = clf2.predict(X_test )
def Functional(f):
if f == 'Typ':
return 1
elif f == 'Min1':
return 2
elif f == 'Min2':
return 3
elif f == 'Mod':
return 4
elif f == 'Maj1':
return 5
elif f == 'Maj2':
return 6
elif f == 'Sev':
return 7
elif f == 'Sal':
return 8
else:
return 0<data_type_conversions>
accuracy_score(y_test, result )
df_all['Functional'] = df_all['Functional'].apply(Functional ).astype(int )<categorify>
print(classification_report(y_test, result))
df_dummy = pd.get_dummies(df_all['GarageType'],prefix='GarageType',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['GarageType']<categorify>
forest = RandomForestClassifier(random_state=42)
 %time forest = forest.fit(X_train, y_train)
 result = forest.predict(X_test )
df_dummy = pd.get_dummies(df_all['SaleType'],prefix='SaleType',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['SaleType']<categorify>
accuracy_score(y_test, result )
df_dummy = pd.get_dummies(df_all['SaleCondition'],prefix='SaleCondition',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['SaleCondition']<categorify>
print(classification_report(y_test, result))
df_dummy = pd.get_dummies(df_all['Heating'],prefix='Heating',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Heating']<categorify>
clf3 = GradientBoostingClassifier(random_state=42)
 %time clf3 = clf3.fit(X_train, y_train)
 result = clf3.predict(X_test )
df_dummy = pd.get_dummies(df_all['KitchenQual'],prefix='KitchenQual',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['KitchenQual']<categorify>
accuracy_score(y_test, result )
df_all['GarageQual'] = df_all['GarageQual'].fillna('TA')
 df_dummy = pd.get_dummies(df_all['GarageQual'],prefix='GarageQual',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['GarageQual']<categorify>
print(classification_report(y_test, result))
df_dummy = pd.get_dummies(df_all['Foundation'],prefix='Foundation',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Foundation']<categorify>
clf = LogisticRegression(random_state=42, solver='lbfgs')
 %time clf = clf.fit(X_train, y_train)
 result = clf.predict(X_test )
df_dummy = pd.get_dummies(df_all['GarageCond'],prefix='GarageCond',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['GarageCond']<categorify>
accuracy_score(y_test, result )
df_dummy = pd.get_dummies(df_all['PavedDrive'],prefix='PavedDrive',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['PavedDrive']<data_type_conversions>
print(classification_report(y_test, result))
df_all['GarageYrBlt']= df_all['GarageYrBlt'].fillna('2003' ).astype(int)
 df_all['TotalBsmtSF'] = df_all['TotalBsmtSF'].fillna('1305' ).astype(int)
 df_all['BsmtFullBath'] = df_all['BsmtFullBath'].fillna('0' ).astype(int)
 df_all['BsmtHalfBath'] = df_all['BsmtHalfBath'].fillna('0' ).astype(int)
 df_all['GarageCars'] = df_all['GarageCars'].fillna('2' ).astype(int)
 df_all['GarageArea'] = df_all['GarageArea'].fillna('420' ).astype(int)
 df_all['PoolArea']=df_all['PoolArea'].astype(int )<drop_column>
clf4 = SVC(random_state=42)
 %time clf4 = clf4.fit(X_train, y_train)
 result = clf4.predict(X_test )
df_all =df_all.drop(['YrSold','MoSold','MiscVal','PoolQC','ScreenPorch','3SsnPorch','EnclosedPorch',
'KitchenAbvGr','CentralAir','Alley','LotConfig','LotShape','LandSlope','RoofStyle',
'BsmtFinSF2','BsmtUnfSF','HeatingQC','Electrical','BsmtFinType2','BsmtFinType1',
'GarageFinish','MiscFeature','FireplaceQu','Fence','Alley','Street','MasVnrType'], axis = 1 )<normalization>
accuracy_score(y_test, result )
Scaler = StandardScaler()
all_scaled = pd.DataFrame(Scaler.fit_transform(df_all))
train_scaled = pd.DataFrame(all_scaled[:1460])
 test_scaled = pd.DataFrame(all_scaled[1460:2920] )<split>
print(classification_report(y_test, result))
X_train = train_scaled
y_train = y['SalePrice']
X_train, X_test, y_train, y_test = train_test_split(
 X_train, y_train, test_size=0.15, random_state=0 )<define_search_space>
def prevendo_noticias(string, model):
to_array=[]
to_array.append(review_to_words(string))
sample_final=vectorizer.transform(to_array)
 sample_final=sample_final.toarray()
result = model.predict(sample_final)
 if result[0] == 1:
label = 'Fake News'
else:
label = 'Verdadeira'
return label, string
XGBoost
param_xgb = {"max_depth": [1,3,5,10,100],
"learning_rate" : [0.0001,0.001,0.01],
"min_child_weight" : [1, 3, 5, 10],
"n_estimators": [1, 10, 100, 1000],
"subsample": [0.5,0.75,0.9],
"gamma":[0,0.1,0.2],
"eta": [0.3,0.15,0.10]}
gs_xgb = GridSearchCV(XGBRegressor() ,
param_xgb,
cv=4,
verbose=True,
n_jobs=-1)
 gs_xgb.fit(X_train, y_train)
 print(gs_xgb.best_estimator_ )<choose_model_class>
prevendo_noticias('Bolsonaro pessoalmente incendêia a amazonia e mata as girafas', forest )
LightGBM
param_lgb = {"max_depth": [1, 3, 5, 10, 25, 50, 75],
"learning_rate" : [0.001,0.01,0.05,0.1],
"num_leaves": [1, 10, 100,1000],
"n_estimators": [1, 3, 10, 100, 1000]}
gs_lgb = GridSearchCV(LGBMRegressor() ,
param_lgb,
cv=4,
verbose=True,
n_jobs=-1)
 gs_lgb.fit(X_train, y_train)
 print(gs_lgb.best_estimator_ )<choose_model_class>
prevendo_noticias('Jornalista joga água benta em Temer e ele admite que impeachment foi golpe', forest )
XGB = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, eta=0.3, gamma=0,
gpu_id=-1, importance_type='gain', interaction_constraints='',
learning_rate=0.01, max_delta_step=0, max_depth=5,
min_child_weight=1,monotone_constraints='() ',n_estimators=1000,
n_jobs=0, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=0.5,
tree_method='exact', validate_parameters=1, verbosity=None)
 XGB.fit(X_train,y_train )<train_model>
model_final = RandomForestClassifier(random_state=42)
model_lgb = LGBMRegressor(learning_rate=0.05, max_depth=3, n_estimators=1000, num_leaves=10)
 model_lgb.fit(X_train , y_train )<train_model>
param_grid = {
'n_estimators': [100, 300, 500, 800, 1000],
'criterion': ['gini', 'entropy'],
'bootstrap': [True, False]
}
votingC = VotingRegressor(estimators=[('xgb_r', XGB),('lgb_r', model_lgb)],n_jobs=4)
 votingC = votingC.fit(X_train, y_train )<compute_test_metric>
CV_rf = GridSearchCV(estimator=model_final, param_grid=param_grid, cv= 5, scoring='accuracy', n_jobs=-1)
 CV_rf = CV_rf.fit(train_data_features, train_y )
print("Training score:",XGB.score(X_train,y_train),"Test Score:",XGB.score(X_test,y_test))
print("Training score:",model_lgb.score(X_train,y_train),"Test Score:",model_lgb.score(X_test,y_test))
print("Training score:",votingC.score(X_train,y_train),"Test Score:",votingC.score(X_test,y_test))<predict_on_test>
CV_rf.best_params_
y_pred_voting = pd.DataFrame(votingC.predict(test_scaled))<create_dataframe>
model_fit = RandomForestClassifier(random_state=42, bootstrap= True, criterion= 'entropy', n_estimators= 800 )
y_pred=pd.DataFrame()
y_pred['SalePrice'] = y_pred_voting[0]
y_pred['Id'] = df_test['Id']<save_to_csv>
%time model_fit = model_fit.fit(X_train, y_train)
 result = model_fit.predict(X_test )
y_pred.to_csv('submission.csv',index=False )<import_modules>
accuracy_score(y_test, result )
import pandas as pd
import gc<load_from_csv>
%time model_final = model_fit.fit(train_data_features, train_y )
Train = pd.read_csv('.. /input/texts-classification-iad-hse-intro-2020/train.csv' )<load_from_csv>
test = pd.read_csv('.. /input/fakenewsvortexbsb/sample_submission.csv', sep=';', error_bad_lines=False, quoting=3);
Test = pd.read_csv('.. /input/texts-classification-iad-hse-intro-2020/test.csv' )<count_missing_values>
clean_test_review=[]
for i in range(0,num_reviews):
clean_test_review.append(review_to_words(test['Manchete'][i]))
Train.isnull().sum()<count_missing_values>
test_data_features = vectorizer.transform(clean_test_review)
 test_data_features=test_data_features.toarray()
Test.isnull().sum()<data_type_conversions>
result_test = model_final.predict(test_data_features )
<correct_missing_values><EOS> 
minha_sub = pd.DataFrame({'index': test.index, 'Category': result_test})
 minha_sub.to_csv('submission.csv', index=False )