Kaggle competition: Tweet Sentiment Analysis
<SOS> metric: auc Kaggle data source: tweet-sentiment-analysis<create_dataframe>
train = pd.read_csv('.. /input/tweet-sentiment-analysis/train.csv')
 test = pd.read_csv('.. /input/tweet-sentiment-analysis/test.csv' )
fig = plt.figure(figsize=(22, 6))
compare_df = pd.DataFrame({'Y_pred':Y_pred[:500],
'Y_valid':Y_valid_expm1[:500],
'Y_pred_correct':Y_pred_correct[:500],
'Y_pred_correct_add':Y_pred_correct_add[:500]})
 sns.lineplot(data=compare_df)
 plt.show()<compute_test_metric>
print(f"There are {len(train)} tweets in the training data.")
 positive = sum(train.target)/len(train)
 print(f"{positive:.2f}% of the tweets are positive.")
 avg_len = sum([len(text)for text in train.text])/len(train)
 print(f"The average length of the tweets is {avg_len:.1f} characters." )
print('Expm1 after and Correct Factor RMSE = ', np.sqrt(sklearn.metrics.mean_squared_error(Y_valid_expm1,Y_pred_correct)))
 print('Expm1 after and Correct Add RMSE = ', np.sqrt(sklearn.metrics.mean_squared_error(Y_valid_expm1,Y_pred_correct_add)) )<save_to_csv>
count_vect = CountVectorizer()
count_vect.fit(train['text'])
 bow_train = count_vect.transform(train['text'])
 bow_test = count_vect.transform(test['text'] )
Y_test = np.expm1(model.predict(X_test)).clip(0, 20)+np.mean(Y_valid_expm1-Y_pred)
 submission = pd.DataFrame({
"ID": test_data.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False )<load_from_csv>
count_vect = CountVectorizer(max_features=15000)
 count_vect.fit(train['text'])
 bow_train = count_vect.transform(train['text'])
 bow_test = count_vect.transform(test['text'] )
sales_train = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sales_train.csv", parse_dates=['date'], infer_datetime_format=False, dayfirst=True)
 test = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 item_categories = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/item_categories.csv")
 items = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/items.csv")
 shops = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/shops.csv")
 print("Sales_train")
 display(sales_train.head(10))
print("Test")
 display(test.head(10))
print("Item_categories")
 display(item_categories.head(10))
print("Items")
 display(items.head(10))
print("Shops")
 display(shops.head(1))
def downcast_dtypes(df):
float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float16)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df
test['date_block_num'] = 34
test['date'] = datetime.datetime(2015, 11, 1, 0, 0, 0)
 all_data = pd.concat([sales_train, test], axis = 0, sort=False)
 all_data['date_block_num'].fillna(34, inplace=True)
 all_data['in_test'] = 0
all_data.loc[all_data.date_block_num == 34, 'in_test'] = 1
new_items =(set(test['item_id'].unique())- set(sales_train['item_id'].unique()))
all_data.loc[all_data['item_id'].isin(new_items), 'is_new_item'] = 1
all_data.fillna(0, inplace=True)
 all_data = downcast_dtypes(all_data)
 all_data = all_data.reset_index()
display(all_data.head(10))
print("Train set size: ", len(sales_train))
print("Test set size: ", len(test))
print("Item categories set size: ", len(item_categories))
print("Items set size: ", len(items))
print("Shops set size: ", len(shops))
print("All data size: ", len(all_data))
print("Duplicates in train dataset: ", len(sales_train[sales_train.duplicated() ]))<drop_column>
model = LogisticRegression(max_iter=1000 ).fit(bow_train, train.target )
all_data = all_data.drop(all_data[all_data['item_price']>100000].index)
 all_data = all_data.drop(all_data[all_data['item_cnt_day']>1100].index)
 sales_train = sales_train.drop(sales_train[sales_train['item_price']>100000].index)
 sales_train = sales_train.drop(sales_train[sales_train['item_cnt_day']>1100].index)
 all_data.loc[all_data['shop_id'] == 11,'shop_id'] = 10
all_data.loc[all_data['shop_id'] == 57,'shop_id'] = 0
all_data.loc[all_data['shop_id'] == 58,'shop_id'] = 1
sales_train.loc[sales_train['shop_id'] == 11,'shop_id'] = 10
sales_train.loc[sales_train['shop_id'] == 57,'shop_id'] = 0
sales_train.loc[sales_train['shop_id'] == 58,'shop_id'] = 1
test.loc[test['shop_id'] == 11,'shop_id'] = 10
test.loc[test['shop_id'] == 57,'shop_id'] = 0
test.loc[test['shop_id'] == 58,'shop_id'] = 1
all_data.loc[all_data['item_price'] < 0, 'item_price'] = all_data[(all_data['shop_id'] == 32)& 
(all_data['item_id'] == 2973)& 
(all_data['date_block_num'] == 4)& 
(all_data['item_price'] > 0)].item_price.median()
print("Raw data length: ",len(sales_train), ", post-outliers length: ", len(all_data))<merge>
predictions = model.predict_proba(bow_test)
 predictions = [probs[1] for probs in predictions]
ts = time.time()
def enrich_data(all_data, items, shops, item_categories):
count_data = all_data.groupby(['shop_id', 'item_id', 'date_block_num', 'in_test', 'is_new_item'])['item_cnt_day'].sum().rename('item_cnt_month' ).reset_index()
price_data = all_data.groupby(['shop_id', 'item_id', 'date_block_num', 'in_test', 'is_new_item'])['item_price'].median().rename('item_price_median' ).reset_index()
all_data = pd.merge(count_data, price_data, on=['shop_id', 'item_id', 'in_test', 'date_block_num', 'is_new_item'], how='left')
 all_data = all_data.join(items, on='item_id', rsuffix='_item')
 all_data = all_data.join(shops, on='shop_id', rsuffix='_shop')
 all_data = all_data.join(item_categories, on='item_category_id', rsuffix='_item_category')
 all_data = all_data.drop(columns=['item_id_item', 'shop_id_shop', 'item_category_id_item_category', 'item_name'])
 categories_split = all_data['item_category_name'].str.split('-')
 all_data['main_category'] = categories_split.map(lambda row: row[0].strip())
 all_data['secondary_category'] = categories_split.map(lambda row: row[1].strip() if(len(row)>1)else 'N/A')
 all_data['city'] = all_data['shop_name'].str.split(' ' ).map(lambda row: row[0])
 all_data.loc[all_data.city == '!Якутск', 'city'] = 'Якутск'
encoder = sklearn.preprocessing.LabelEncoder()
all_data['city_label'] = encoder.fit_transform(all_data['city'])
 all_data['main_category_label'] = encoder.fit_transform(all_data['main_category'])
 all_data['secondary_category_label'] = encoder.fit_transform(all_data['secondary_category'])
 all_data = all_data.drop(['city', 'shop_name', 'item_category_name', 'main_category', 'secondary_category'], axis = 1)
 def price_category(row):
if row.item_price_median<5.:
val = 1
elif row.item_price_median<10.:
val = 2
elif row.item_price_median<100.:
val = 3
elif row.item_price_median<200.:
val = 4
elif row.item_price_median<300.:
val = 5
elif row.item_price_median<500.:
val = 6
elif row.item_price_median<1000.:
val = 7
elif row.item_price_median>1000.:
val = 8
else:
val = 0
return val
all_data['price_cat'] = all_data.apply(price_category, axis=1)
 all_data = downcast_dtypes(all_data)
 return all_data
all_data2 = enrich_data(all_data, items, shops, item_categories)
 items_prices = all_data2[['item_id', 'shop_id', 'date_block_num', 'item_price_median', 'price_cat']]
time.time() - ts
all_data2.head()<create_dataframe>
submission = pd.read_csv('.. /input/tweet-sentiment-analysis/sample_submission.csv')
 submission['Predicted'] = predictions
<data_type_conversions><EOS> 
submission.to_csv('/kaggle/working/submission.csv', index=False )