Kaggle competition: Digit Recognizer
train.drop(train[train["price"] <= 0].index, inplace=True)
 train.reset_index(inplace=True, drop=True )<feature_engineering>
from keras.utils.np_utils import to_categorical
def process_data(df):
df[["First_category", "Second_category", "Third_category"]] = df["category_name"].str.split('/', 2, expand=True)
 df.drop("category_name", axis=1, inplace=True)
 df["item_description"] = df["item_description"].replace({"No description yet": np.nan})
 df["Category_was_missing"] = df["First_category"].isna()
df["Brand_was_missing"] = df["brand_name"].isna()
df["Description_was_missing"] = df["item_description"].isna()
for i in ["brand_name", "First_category", "Second_category", "Third_category", "item_description"]:
df[i] = df[i].fillna("missing")
 df["Price_was_in_description"] = df["item_description"].str.contains("\[rm\]")
 df["Price_was_in_name"] = df["name"].str.contains("\[rm\]")
 df["descr_len"] = df["item_description"].str.len()
df["descr_len"] = pd.cut(df["descr_len"], np.arange(0, 1055, 5), right=False)
 df["descr_len"] = df["descr_len"].astype("string")
 stop_words = set(stopwords.words("english"))
for column in ["item_description", "brand_name", "name"]:
processed_column = []
for text_row in df[column]:
text_row = text_row.replace('[rm]', '')
 text_row = re.sub('[^A-Za-z0-9]+', ' ', text_row)
 if column != "brand_name":
text_row = ' '.join(word for word in text_row.lower().split() if word not in stop_words)
 processed_column.append(text_row.strip())
 df[column] = processed_column
for column in ["First_category", "Second_category", "Third_category"]:
processed_column = []
for text_row in df[column]:
text_row = text_row.replace(' ','')
 text_row = text_row.replace('&','_')
 text_row = re.sub('[^A-Za-z0-9_]+', ' ', text_row)
 processed_column.append(text_row.lower().strip())
 df[column] = processed_column
return df<create_dataframe>
training_dataset = pd.read_csv("/kaggle/input/train.csv")
 testing_dataset = pd.read_csv("/kaggle/input/test.csv" )
train = process_data(train )<prepare_x_and_y>
Y_train_init = training_dataset['label']
X_train_init = training_dataset.drop(labels=['label'], axis=1 )
def make_tokens_count_plot(count_vercorizer, matrix, column_name, set_name="train"):
if set_name == "train":
color = "lightcoral"
else:
color = "steelblue"
df = pd.DataFrame()
df["tokens"] = count_vercorizer.get_feature_names()
df["counts"] = np.asarray(matrix.sum(axis=0)) [0]
x = df.sort_values("counts", axis=0, ascending=False ).head(50)["tokens"]
y = df.sort_values("counts", axis=0, ascending=False ).head(50)["counts"]
fig, ax = plt.subplots(figsize=(15, 6))
ax.bar(x, y, color=color, edgecolor="black")
 ax.set_title(f"Most popular words in the {column_name} column of the {set_name} dataset", fontsize=20, pad=15)
 ax.set_ylabel("Count", fontsize=14, labelpad=15)
 ax.set_xlabel("Word", fontsize=14, labelpad=10)
 ax.set_xticks(x)
 ax.set_xticklabels(x, rotation = 60, ha="right", rotation_mode='anchor')
 ax.tick_params(axis="x", labelsize=14)
 ax.tick_params(axis="y", labelsize=14)
 ax.grid(axis="y")
 ax.margins(0.025, 0.05)
 plt.show() ;<split>
def get_transformed_train_valid_data(df, y, cat_features):

 X_train, X_valid, y_train, y_valid = train_test_split(df, y, test_size=0.2, random_state=42)
 label_binarizers = []
binarized_columns = []
count_vercorizers = []
vectorized_columns = []
for column in cat_features:
binarizer = LabelBinarizer(sparse_output=True)
 binarized_column = binarizer.fit_transform(X_train[column])
 label_binarizers.append(binarizer)
 binarized_columns.append(binarized_column)
 vectorizer = CountVectorizer(min_df=7, max_features=20000)
 vectorized_column = vectorizer.fit_transform(X_train["name"])
 count_vercorizers.append(vectorizer)
 vectorized_columns.append(vectorized_column)
 make_tokens_count_plot(vectorizer, vectorized_column, "name", "train")
 vectorizer = CountVectorizer(min_df=15, ngram_range=(1, 2), max_features=60000)
 vectorized_column = vectorizer.fit_transform(X_train["item_description"])
 count_vercorizers.append(vectorizer)
 vectorized_columns.append(vectorized_column)
 make_tokens_count_plot(vectorizer, vectorized_column, "item_description", "train")
 vectorizer = CountVectorizer(min_df=30, ngram_range=(3, 3), max_features=10000)
 vectorized_column = vectorizer.fit_transform(X_train["item_description"])
 count_vercorizers.append(vectorizer)
 vectorized_columns.append(vectorized_column)
 make_tokens_count_plot(vectorizer, vectorized_column, "item_description", "train")
 print(f"Name columns vectorized shape is {vectorized_columns[0].shape}")
 print(f"Item_description columns vectorized shape is {vectorized_columns[1].shape}")
 print(f"Item_description columns vectorized(1,3)shape is {vectorized_columns[2].shape}")
 X_train_stack = hstack(( binarized_columns[0], binarized_columns[1], binarized_columns[2],
binarized_columns[3], binarized_columns[4], binarized_columns[5],
binarized_columns[6], binarized_columns[7], binarized_columns[8],
binarized_columns[9], binarized_columns[10], binarized_columns[11],
vectorized_columns[0], vectorized_columns[1], vectorized_columns[2])).tocsr()
X_valid_stack = get_transformed_test_data(X_valid, cat_features, label_binarizers, count_vercorizers)
 return X_train_stack, X_valid_stack, y_train, y_valid, label_binarizers, count_vercorizers
def get_transformed_test_data(df, cat_features, label_binarizers, count_vercorizers):

 binarized_columns = []
vectorized_columns = []
for num, column in enumerate(cat_features):
binarizer = label_binarizers[num]
binarized_column = binarizer.transform(df[column])
 binarized_columns.append(binarized_column)
 vectorizer = count_vercorizers[0]
vectorized_column = vectorizer.transform(df["name"])
 vectorized_columns.append(vectorized_column)
 make_tokens_count_plot(vectorizer, vectorized_column, "name", "valid/test")
 vectorizer = count_vercorizers[1]
vectorized_column = vectorizer.transform(df["item_description"])
 vectorized_columns.append(vectorized_column)
 make_tokens_count_plot(vectorizer, vectorized_column, "item_description", "valid/test")
 vectorizer = count_vercorizers[2]
vectorized_column = vectorizer.transform(df["item_description"])
 vectorized_columns.append(vectorized_column)
 make_tokens_count_plot(vectorizer, vectorized_column, "item_description", "valid/test")
 print(f"Name columns vectorized shape is {vectorized_columns[0].shape}")
 print(f"Item_description columns vectorized shape is {vectorized_columns[1].shape}")
 print(f"Item_description columns vectorized(1,3)shape is {vectorized_columns[2].shape}")
 X_test_stack = hstack(( binarized_columns[0], binarized_columns[1], binarized_columns[2],
binarized_columns[3], binarized_columns[4], binarized_columns[5],
binarized_columns[6], binarized_columns[7], binarized_columns[8],
binarized_columns[9], binarized_columns[10], binarized_columns[11],
vectorized_columns[0], vectorized_columns[1], vectorized_columns[2])).tocsr()
return X_test_stack
def get_transformed_train_data(df, cat_features):

 label_binarizers = []
binarized_columns = []
count_vercorizers = []
vectorized_columns = []
for column in cat_features:
binarizer = LabelBinarizer(sparse_output=True)
 binarized_column = binarizer.fit_transform(df[column])
 label_binarizers.append(binarizer)
 binarized_columns.append(binarized_column)
 vectorizer = CountVectorizer(min_df=7, max_features=20000)
 vectorized_column = vectorizer.fit_transform(df["name"])
 count_vercorizers.append(vectorizer)
 vectorized_columns.append(vectorized_column)
 make_tokens_count_plot(vectorizer, vectorized_column, "name", "train")
 vectorizer = CountVectorizer(min_df=15, ngram_range=(1, 2), max_features=60000)
 vectorized_column = vectorizer.fit_transform(df["item_description"])
 count_vercorizers.append(vectorizer)
 vectorized_columns.append(vectorized_column)
 make_tokens_count_plot(vectorizer, vectorized_column, "item_description", "train")
 vectorizer = CountVectorizer(min_df=30, ngram_range=(3, 3), max_features=10000)
 vectorized_column = vectorizer.fit_transform(df["item_description"])
 count_vercorizers.append(vectorizer)
 vectorized_columns.append(vectorized_column)
 make_tokens_count_plot(vectorizer, vectorized_column, "item_description", "train")
 print(f"Name columns vectorized shape is {vectorized_columns[0].shape}")
 print(f"Item_description columns vectorized shape is {vectorized_columns[1].shape}")
 print(f"Item_description columns vectorized(1,3)shape is {vectorized_columns[2].shape}")
 X_train_stack = hstack(( binarized_columns[0], binarized_columns[1], binarized_columns[2],
binarized_columns[3], binarized_columns[4], binarized_columns[5],
binarized_columns[6], binarized_columns[7], binarized_columns[8],
binarized_columns[9], binarized_columns[10], binarized_columns[11],
vectorized_columns[0], vectorized_columns[1], vectorized_columns[2])).tocsr()
return X_train_stack, label_binarizers, count_vercorizers<drop_column>
Y_train = training_dataset['label']
X_train = training_dataset.drop(labels=['label'], axis=1)
 nums = 10
cat_features = ["item_condition_id", "First_category", "Second_category", "Third_category",
"shipping", "brand_name", "Description_was_missing", "Price_was_in_name",
"Price_was_in_description", "Brand_was_missing", "Category_was_missing",
"descr_len"]
X_train, label_binarizers, count_vercorizers = get_transformed_train_data(train.drop(["train_id", "price"], axis=1), cat_features)
 y_train = np.log1p(train["price"] )<drop_column>
X_test = testing_dataset.values
for column in train.columns:
train.drop(column, axis=1, inplace=True)
 del train
gc.collect()<compute_test_metric>
X_train = X_train/255.0
def rmsle(y_true, y_preds):
return np.sqrt(mean_squared_log_error(y_true, y_preds))
def get_scores(model, X_train, X_valid, y_train, y_valid):
train_preds = model.predict(X_train)
 val_preds = model.predict(X_valid)
 scores = {"Training MAE": mean_absolute_error(y_train, train_preds),
"Validation MAE": mean_absolute_error(y_valid, val_preds),
"Training RMSLE": rmsle(y_train, train_preds),
"Validation RMSLE": rmsle(y_valid, val_preds),
"Training R^2": model.score(X_train, y_train),
"Validation R^2": model.score(X_valid, y_valid)}
return scores<train_model>
X_test = X_test/ 255.0
%%time
model = SGDRegressor(alpha=0.000001,
penalty="l2",
random_state=42,
loss="epsilon_insensitive",
max_iter=50000)
 model.fit(X_train, y_train )<drop_column>
Y_train = to_categorical(Y_train, num_classes=nums )
del X_train, y_train
gc.collect()<create_dataframe>
random_seed = 2
preds = pd.DataFrame(columns = ["test_id", "price"])
 preds["test_id"] = test["test_id"]<categorify>
from sklearn.model_selection import train_test_split
test = process_data(test.drop("test_id", axis=1))
X_test = get_transformed_test_data(test, cat_features, label_binarizers, count_vercorizers )<save_to_csv>
from sklearn.model_selection import train_test_split
preds["price"] = np.expm1(model.predict(X_test))
preds.to_csv('submission.csv', index=False)
 preds.head()<load_pretrained>
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, random_state=random_seed, test_size = 0.3 )
!apt-get install p7zip
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/train.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/test.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/sample_submission.csv.7z<load_from_csv>
from keras import Sequential
from keras.layers import Dense, MaxPooling2D, Dropout, Flatten, Conv2D, BatchNormalization
from keras.callbacks import ReduceLROnPlateau, LearningRateScheduler
from keras.preprocessing.image import ImageDataGenerator
!unzip /kaggle/input/mercari-price-suggestion-challenge/sample_submission_stg2.csv.zip
!unzip /kaggle/input/mercari-price-suggestion-challenge/test_stg2.tsv.zip<import_modules>
batch_size = 65
epochs = 23
model = Sequential()
model.add(Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same', input_shape=(28, 28, 1)))
 model.add(BatchNormalization(scale=False,center=True))
model.add(MaxPooling2D(padding='same', pool_size=(2, 2)))
 model.add(Conv2D(filters=32, kernel_size=(5, 5), activation='relu', padding='same'))
model.add(MaxPooling2D(padding='same', pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters=64, kernel_size=(5, 5), activation='relu', padding='same'))
model.add(MaxPooling2D(padding='same', pool_size=(2, 2)))
 model.add(Conv2D(filters=64, kernel_size=(7, 7), activation='relu', padding='same'))
model.add(MaxPooling2D(padding='same', pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters=128, kernel_size=(5, 5), activation='relu', padding='same'))
model.add(MaxPooling2D(padding='same', pool_size=(2, 2)))
 model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(Dropout(0.25))
model.add(BatchNormalization(scale=False,center=True))
model.add(Dense(nums, activation='softmax'))
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import math
from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer<load_from_csv>
model.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=['accuracy'])
train = pd.read_table('train.tsv')
 test = pd.read_table('test_stg2.tsv')
 print(train.shape)
 print(test.shape )<count_missing_values>
t = 1
def decide_f(t):
if t < 5:
f =.5
else:
f =.9
t = t + 1
return(f)
 reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=decide_f(t),verbose=1,
patience=2, min_lr=0.00000000001 )
train.isnull().sum()<categorify>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=15,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False )
def handle_missing(dataset):
dataset.category_name.fillna(value="missing", inplace=True)
 dataset.brand_name.fillna(value="missing", inplace=True)
 dataset.item_description.fillna(value="missing", inplace=True)
 return(dataset )<correct_missing_values>
datagen.fit(X_train)
 h = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 1, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[reduce_lr],)
 final_loss, final_acc = model.evaluate(X_val, Y_val, verbose=0)
 print("Final loss: {0:.6f}, final accuracy: {1:.6f}".format(final_loss, final_acc))
train = handle_missing(train)
 test = handle_missing(test )<categorify>
prediction = model.predict(X_test)
def transform_category_name(category_name):
try:
sub1, sub2, sub3 = category_name.split('/')
 return sub1, sub2, sub3
except:
return "none", "none", "none"
train['subcat_1'], train['subcat_2'], train['subcat_3'] = zip(*train['category_name'].apply(transform_category_name))
test['subcat_1'], test['subcat_2'], test['subcat_3'] = zip(*test['category_name'].apply(transform_category_name))
train.head(10 )<count_unique_values>
label = np.argmax(prediction, axis=1 )
print("There are %d unique subcat_1." % train['subcat_1'].nunique())
 print("There are %d unique subcat_2." % train['subcat_2'].nunique())
 print("There are %d unique subcat_3." % train['subcat_3'].nunique() )<count_values>
test_id = np.reshape(range(1, len(prediction)+ 1), label.shape )
<categorify><EOS> 
my_submission = pd.DataFrame({'ImageId': test_id, 'Label': label})
 my_submission.to_csv('submission.csv', index=False )