Kaggle competition: San Francisco Crime Classification
<SOS> metric: MulticlassLoss Kaggle data source: san-francisco-crime-classification<set_options>
from IPython.display import Image
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 100 )<load_from_csv>
Image(filename=".. /input/sf-picture/sf1.jpg" )
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 length_train = len(train.index)
 length_train<count_unique_values>
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
sns.set_style("white")
 rcParams['figure.figsize'] =(8,4)
df_mislabeled = train.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']
index_misl = df_mislabeled.index.tolist()
lenght = len(index_misl)
 print(f"There are {lenght} equivalence classes with mislabelling" )<sort_values>
df = pd.read_csv(".. /input/sf-crime/train.csv.zip",dtype={"X":np.float32,"Y":np.float32} )
train_nu_target = train[train['text'].isin(index_misl)].sort_values(by = 'text')
 train_nu_target.head(60 )<define_variables>
print(df.duplicated(keep=False ).value_counts())
 df = df.drop_duplicates()
num_records = train_nu_target.shape[0]
length = len(index_misl)
 print(f"There are {num_records} records in train set which generate {lenght} equivalence classes with mislabelling(raw text, no cleaning)" )<groupby>
def convert_dataframe(df):

 df["Dates"] = pd.to_datetime(df["Dates"],infer_datetime_format=True)
 df['Date'] = df['Dates'].dt.date
df["Year"] = df["Dates"].dt.year.astype(np.int32)
 df["Month"] = df["Dates"].dt.month.astype(np.int32)
 df["Day"] = df["Dates"].dt.day.astype(np.int32)
 df["Hour"] = df["Dates"].dt.hour.astype(np.int32)
 df["Minute"] = df["Dates"].dt.minute.astype(np.int32)
 df["Day_week_numeric"] = df["Dates"].dt.dayofweek.astype(np.int32)
 df["Weekend"]= np.where(( df["Day_week_numeric"] >= 4)&(df["Day_week_numeric"] <=6),1,0)
 df["count_days"] =(df['Date'] - df['Date'].min() ).apply(lambda x: x.days)
 df["Block"] = df.Address.str.contains("Block" ).astype(np.int32)
 df = df.drop(["Date","Address"],axis=1)
 return df
copy = train_nu_target.copy()
classes = copy.groupby('text' ).agg({'keyword':np.size, 'target':np.mean} ).rename(columns={'keyword':'Number of records in train set', 'target':'Target mean'})
 classes.sort_values('Number of records in train set', ascending=False ).head(20 )<groupby>
df_date = convert_dataframe(df )
majority_df = train_nu_target.groupby(['text'])['target'].mean() 
<categorify>
label_cat = LabelEncoder()
df_date["Category_encode"] = label_cat.fit_transform(df_date.Category)
 label_dist = LabelEncoder()
df_date["PdDistric_encode"] = label_dist.fit_transform(df_date.PdDistrict)
def relabel(r, majority_index):
ind = ''
if r['text'] in majority_index:
ind = r['text']
if majority_df[ind] <= 0.5:
return 0
else:
return 1
else:
return r['target']<feature_engineering>
df_outliers = df_date.loc[df_date.Y < 90.].copy()
train['target'] = train.apply(lambda row: relabel(row, majority_df.index), axis = 1 )<sort_values>
X = df_outliers.drop(["Dates","Category","Descript","DayOfWeek","PdDistrict","Resolution","Category_encode"],axis=1 ).copy()
y = df_outliers["Category_encode"]
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.25, random_state = 21)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_val = scaler.transform(X_val)
 kmeans = KMeans(n_clusters=6,random_state=0 ).fit(X_train)
new_df = train[train['text'].isin(majority_df.index)].sort_values(['target', 'text'], ascending = [False, True])
 new_df.head(15 )<count_unique_values>
X_train_df = pd.DataFrame(X_train)
 X_val_df = pd.DataFrame(X_val)
 X_train_df["Kmean"] = kmeans.labels_
X_val_df["Kmean"] = kmeans.predict(X_val )
df_mislabeled = train.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']
index_misl = df_mislabeled.index.tolist()
len(index_misl )<load_pretrained>
classifier = RandomForestClassifier(n_jobs = -1,random_state =50,max_depth=10,max_features="auto",min_samples_split=4)
 classifier.fit(X_train_df, y_train )
use = hub.load("https://tfhub.dev/google/universal-sentence-encoder-multilingual-large/3" )<categorify>
predict_proba = classifier.predict_proba(X_val_df)
 log_loss(y_val,predict_proba )
X_train = []
for r in tqdm(train.text.values):
emb = use(r)
 review_emb = tf.reshape(emb, [-1] ).numpy()
X_train.append(review_emb)
 X_train = np.array(X_train)
 y_train = train.target.values
X_test = []
for r in tqdm(test.text.values):
emb = use(r)
 review_emb = tf.reshape(emb, [-1] ).numpy()
X_test.append(review_emb)
 X_test = np.array(X_test )<split>
test_data = pd.read_csv(".. /input/sf-crime/test.csv.zip" )
train_arrays, test_arrays, train_labels, test_labels = train_test_split(X_train,
y_train,
random_state =42,
test_size=0.20 )<train_on_grid>
test_data_transformed = convert_dataframe(test_data)
 test_data_transformed["PdDistric_encode"] = label_dist.fit_transform(test_data_transformed.PdDistrict)
 test_data_final = test_data_transformed.drop(["DayOfWeek","PdDistrict","Dates",
"Id"],axis=1 ).copy()
def svc_param_selection(X, y, nfolds):
Cs = [1.07]
gammas = [2.075]
param_grid = {'C': Cs, 'gamma' : gammas}
grid_search = GridSearchCV(SVC(kernel='rbf'), param_grid, cv=nfolds, n_jobs=8)
 grid_search.fit(X, y)
 grid_search.best_params_
return grid_search
model = svc_param_selection(train_arrays,train_labels, 5 )<find_best_params>
test_data_scaler = scaler.transform(test_data_final )
model.best_params_<predict_on_test>
test_data_final = pd.DataFrame(test_data_scaler)
 test_data_final["Kmean"] = kmeans.predict(test_data_final)
pred = model.predict(test_arrays )<compute_test_metric>
test_data_pred_proba = classifier.predict_proba(test_data_final)
 keys = label_cat.classes_
cm = confusion_matrix(test_labels,pred)
 cm<compute_test_metric>
result = pd.DataFrame(data=test_data_pred_proba,columns=keys)
 result.head(3 )
<predict_on_test><EOS> 
result.to_csv(path_or_buf="classifier_sf.csv",index=True, index_label = 'Id' )