Kaggle competition: AI Academy Intermediate Class Competition 1
corr_y.sort_values('corr_y' )<feature_engineering>
TRAIN_PATH = os.path.join(".. /input/ai-academy-intermediate-class-competition-1", "BBC News Train.csv")
 df = pd.read_csv(TRAIN_PATH )
gc['Age'] = np.round(gc['Age'].fillna(gc['Age'].mean()))<count_missing_values>
df['category_id'] = df['Category'].factorize() [0]
df['category_id'][0:10]
gc.Age.isnull().sum()<feature_engineering>
category_id_df = df[['Category', 'category_id']].drop_duplicates().sort_values('category_id' )
df['Age']=gc['Age']<feature_engineering>
category_to_id = dict(category_id_df.values)
 id_to_category = dict(category_id_df[['category_id', 'Category']].values )
df['Age']=pd.cut(df['Age'], 5, labels=False )<count_missing_values>
df.groupby('Category' ).category_id.count()
df.isnull().sum()<install_modules>
tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', encoding='latin-1', ngram_range=(1, 2), stop_words='english')
 features = tfidf.fit_transform(df.Text ).toarray()
labels = df.category_id
pip install --upgrade optuna<set_options>
sorted(category_to_id.items() )
gc=df<categorify>
N = 3
for Category, category_id in sorted(category_to_id.items()):
features_chi2 = chi2(features, labels == category_id)
 indices = np.argsort(features_chi2[0])
 feature_names = np.array(tfidf.get_feature_names())[indices]
unigrams = [v for v in feature_names if len(v.split(' ')) == 1]
bigrams = [v for v in feature_names if len(v.split(' ')) == 2]
print("
print(".Most correlated unigrams: 
.{}".format(' 
.'.join(unigrams[-N:])))
 print(".Most correlated bigrams: 
.{}".format(' 
.'.join(bigrams[-N:])) )
gc['Fare']=pd.cut(gc['Fare'], 5, labels=False)
 column = 'Cabin'
gc[column] = gc[column].fillna(0)
 gc[column] = gc[column].str.extract('([A-Za-z]+)', expand = False)
 column = 'mrms'
gc[column] = gc['Name'].str.extract('([A-Za-z]+)\.', expand = False)
 gc[column] = gc[column].replace(['Col', 'Mlle', 'Major','Countess','Lady', 'Sir', 'Dona', 'Capt', 'Jonkheer', 'Don', 'Mme', 'Ms', 'Dr', 'Rev'], 'Other')
 le = LabelEncoder()
gc['Ticket_head']=le.fit_transform(gc['Ticket_head'])
 gc['Ticket_num']=le.fit_transform(gc['Ticket_num'])
 list_ce = ['Cabin','Embarked','Sex','mrms','Fare']
ce_ohe = ce.OneHotEncoder(cols=list_ce,handle_unknown='impute')
 gc = ce_ohe.fit_transform(gc)
 gc = gc.drop(['PassengerId','Name','Ticket'], axis=1 )<define_variables>
SAMPLE_SIZE = int(len(features)* 0.3)
 np.random.seed(0)
 indices = np.random.choice(range(len(features)) , size=SAMPLE_SIZE, replace=False)
 projected_features = TSNE(n_components=2, random_state=0 ).fit_transform(features[indices])
train=gc<prepare_x_and_y>
my_id = 0
projected_features[(labels[indices] == my_id ).values]
test_x = train[train['Survived'].isnull() ]
test_x = test_x.drop(['Survived'], axis=1)
 df_result_dropna = train.dropna(subset=['Survived'])
 feature_names = df_result_dropna.drop(['Survived'], axis=1)
 feature_names = list(feature_names.columns)
 train_y = df_result_dropna['Survived'].astype(int)
 train_x = df_result_dropna.drop(['Survived'], axis=1 )<split>
models = [
RandomForestClassifier(n_estimators=200, max_depth=3, random_state=0),
MultinomialNB() ,
LogisticRegression(random_state=0),
]
tr_x, va_x, tr_y, va_y = train_test_split(train_x, train_y,random_state=42,test_size=0.2 )<import_modules>
CV = 5
cv_df = pd.DataFrame(index=range(CV * len(models)))
 entries = []
import optuna.integration.lightgbm as lgb
from sklearn.metrics import accuracy_score,f1_score
from sklearn.metrics import confusion_matrix<train_model>
for model in models:
model_name = model.__class__.__name__
accuracies = cross_val_score(model, features, labels, scoring='accuracy', cv=CV)
 for fold_idx, accuracy in enumerate(accuracies):
entries.append(( model_name, fold_idx, accuracy))
best_params = {}
tuning_history = []
params = {'objective': 'binary','metric': 'binary_logloss'}
trn_data= lgb.Dataset(tr_x, label=tr_y)
 val_data= lgb.Dataset(va_x, label=va_y)
 model = lgb.train(params, trn_data,
valid_sets=[trn_data, val_data],
verbose_eval=0,
best_params=best_params,
tuning_history=tuning_history
)<predict_on_test>
cv_df = pd.DataFrame(entries, columns=['model_name', 'fold_idx', 'accuracy'] )
prediction = np.rint(model.predict(va_x, num_iteration=model.best_iteration))<compute_test_metric>
cv_df.groupby('model_name' ).accuracy.mean()
accuracy = accuracy_score(va_y, prediction)
 best_params = model.params
print("Best params:", best_params)
 print("Accuracy = {}".format(accuracy))
print("Params: ")
 for key, value in best_params.items() :
print(" {}: {}".format(key, value))<predict_on_test>
model = LogisticRegression(random_state=0)
 X_train, X_test, y_train, y_test, indices_train, indices_test = train_test_split(features, labels, df.index, test_size=0.33, random_state=0)
 model.fit(X_train, y_train)
 y_pred_proba = model.predict_proba(X_test)
 y_pred = model.predict(X_test )
pred_x = np.rint(model.predict(test_x, num_iteration=model.best_iteration))<save_to_csv>
for predicted in category_id_df.category_id:
for actual in category_id_df.category_id:
if predicted != actual and conf_mat[actual, predicted] >= 2:
print("'{}' predicted as '{}' : {} examples.".format(id_to_category[actual], id_to_category[predicted], conf_mat[actual, predicted]))
display(df.loc[indices_test[(y_test == actual)&(y_pred == predicted)]]['Text'])
 print('' )
df_sub['Survived'] = pred_x.astype(int)
 df_sub.to_csv('df_sub_pred_x.csv', index=False )<set_options>
model.fit(features, labels )
%matplotlib inline<load_from_csv>
N = 5
for Category, category_id in sorted(category_to_id.items()):
indices = np.argsort(model.coef_[category_id])
 feature_names = np.array(tfidf.get_feature_names())[indices]
unigrams = [v for v in reversed(feature_names)if len(v.split(' ')) == 1][:N]
bigrams = [v for v in reversed(feature_names)if len(v.split(' ')) == 2][:N]
print("
print(".Top unigrams: 
.{}".format(' 
.'.join(unigrams)))
 print(".Top bigrams: 
.{}".format(' 
.'.join(bigrams)) )
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )<prepare_x_and_y>
texts = ["Hooli stock price soared after a dip in PiedPiper revenue growth.",
"Captain Tsubasa scores a magnificent goal for the Japanese team.",
"Merryweather mercenaries are sent on another mission, as government oversight groups call for new sanctions.",
"Beyoncé releases a new album, tops the charts in all of south-east Asia!",
"You won't guess what the latest trend in data analysis is!"]
text_features = tfidf.transform(texts)
 predictions = model.predict(text_features)
 for text, predicted in zip(texts, predictions):
print('"{}"'.format(text))
print(" - Predicted as: '{}'".format(id_to_category[predicted]))
print("" )
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))<create_dataframe>
TEST_PATH = os.path.join(".. /input/bbc-test", "BBC News Test.csv")
 test_df = pd.read_csv(TEST_PATH)
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'欠損比率' :all_data_na})
 missing_data.head(100 )<data_type_conversions>
test_features = tfidf.transform(test_df.Text.tolist())
 Y_pred = model.predict(test_features)
 Y_pred
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )<data_type_conversions>
Y_pred_name =[]
for cat_id in Y_pred :
Y_pred_name.append(id_to_category[cat_id] )
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )<data_type_conversions>
submission = pd.DataFrame({
"ArticleId": test_df["ArticleId"],
"Category": Y_pred_name
} )
<data_type_conversions><EOS> 
submission.to_csv('submission.csv', index=False )