Kaggle competition: PetFinder.my Adoption Prediction
print("Encodings")
 count = CountVectorizer(min_df=NAME_MIN_DF)
 X_name = count.fit_transform(df["name"])
 print("Category Encoders")
 unique_categories = pd.Series("/".join(df["category_name"].unique().astype("str")).split("/")).unique()
count_category = CountVectorizer()
X_category = count_category.fit_transform(df["category_name"])
 print("Descp encoders")
 count_descp = TfidfVectorizer(max_features = MAX_FEAT_DESCP,
ngram_range =(1,3),
stop_words = "english")
 X_descp = count_descp.fit_transform(df["item_description"])
 print("Brand encoders")
 vect_brand = LabelBinarizer(sparse_output=True)
 X_brand = vect_brand.fit_transform(df["brand_name"])
 print("Dummy Encoders")
 X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[[
"item_condition_id", "shipping"]], sparse = True ).values )<concatenate>
train = pd.read_csv(".. /input/train/train.csv")
 test = pd.read_csv(".. /input/test/test.csv")
 breeds = pd.read_csv(".. /input/breed_labels.csv")
 colors = pd.read_csv(".. /input/color_labels.csv")
 states = pd.read_csv(".. /input/state_labels.csv" )
X = scipy.sparse.hstack(( X_dummies,
X_descp,
X_brand,
X_category,
X_name)).tocsr()
print([X_dummies.shape, X_category.shape,
X_name.shape, X_descp.shape, X_brand.shape])
 X_train = X[:nrow_train]
X_test = X[nrow_train:]<train_model>
train['Name'] = train['Name'].fillna('Unnamed')
 test['Name'] = test['Name'].fillna('Unnamed' )
params = {
'learning_rate': 0.75,
'application': 'regression',
'max_depth': 3,
'num_leaves': 100,
'verbosity': -1,
'metric': 'RMSE',
}
train_X, valid_X, train_y, valid_y = train_test_split(X_train, y_train, test_size = 0.1, random_state = 144)
 d_train = lgb.Dataset(train_X, label=train_y, max_bin=8192)
 d_valid = lgb.Dataset(valid_X, label=valid_y, max_bin=8192)
 watchlist = [d_train, d_valid]
model = lgb.train(params, train_set=d_train, num_boost_round=2200, valid_sets=watchlist, \
early_stopping_rounds=50, verbose_eval=100)
 preds = model.predict(X_test)
 model = Ridge(solver = "lsqr", fit_intercept=False)
 print("Fitting Model")
 model.fit(X_train, y_train)
 preds += model.predict(X_test)
 preds /= 2<save_to_csv>
train['No_name'] = 0
train.loc[train['Name'] == 'Unnamed', 'No_name'] = 1
train.loc[train['Name'] == 'No Name Yet', 'No_name'] = 1
test['No_name'] = 0
test.loc[test['Name'] == 'Unnamed', 'No_name'] = 1
test.loc[train['Name'] == 'No Name Yet', 'No_name'] = 1
df_test["price"] = np.expm1(preds)
 df_test[["test_id", "price"]].to_csv("submission_LGBM_Ridge_3.csv", index = False )<load_from_csv>
train['Out_Age'] = 0
train.loc[train['Age'] > 20, 'Out_Age'] = 1
test['Out_Age'] = 0
test.loc[test['Age'] > 20, 'Out_Age'] = 1
NUM_BRANDS = 2500
NAME_MIN_DF = 10
MAX_FEAT_DESCP = 10000
print("Reading in Data")
 df_train2 = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_test2 = pd.read_csv('.. /input/test.tsv', sep='\t')
 df = pd.concat([df_train2, df_test2], 0)
 nrow_train = df_train2.shape[0]
y_train = np.log(df_train2["price"]+1)
 del df_train2
gc.collect()
print(df.memory_usage(deep = True))
df["category_name"] = df["category_name"].fillna("Other" ).astype("category")
 df["brand_name"] = df["brand_name"].fillna("unknown")
 pop_brands = df["brand_name"].value_counts().index[:NUM_BRANDS]
df.loc[~df["brand_name"].isin(pop_brands), "brand_name"] = "Other"
df["item_description"] = df["item_description"].fillna("None")
 df["item_condition_id"] = df["item_condition_id"].astype("category")
 df["brand_name"] = df["brand_name"].astype("category")
 print(df.memory_usage(deep = True))
print("Encodings")
 count = CountVectorizer(min_df=NAME_MIN_DF)
 X_name = count.fit_transform(df["name"])
 print("Category Encoders")
 unique_categories = pd.Series("/".join(df["category_name"].unique().astype("str")).split("/")).unique()
count_category = CountVectorizer()
X_category = count_category.fit_transform(df["category_name"])
 print("Descp encoders")
 count_descp = TfidfVectorizer(max_features = MAX_FEAT_DESCP,
ngram_range =(1,3),
stop_words = "english")
 X_descp = count_descp.fit_transform(df["item_description"])
 print("Brand encoders")
 vect_brand = LabelBinarizer(sparse_output=True)
 X_brand = vect_brand.fit_transform(df["brand_name"])
 print("Dummy Encoders")
 X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[[
"item_condition_id", "shipping"]], sparse = True ).values)
 X = scipy.sparse.hstack(( X_dummies,
X_descp,
X_brand,
X_category,
X_name)).tocsr()
print([X_dummies.shape, X_category.shape,
X_name.shape, X_descp.shape, X_brand.shape])
 X_train = X[:nrow_train]
model = Ridge(solver = "lsqr", fit_intercept=False)
 print("Fitting Model")
 model.fit(X_train, y_train)
 X_test = X[nrow_train:]
preds1 = model.predict(X_test)
 df_test2["price1"] = np.exp(preds1)-1 
<compute_test_metric>
train['Pure_breed'] = 0
train.loc[train['Breed2'] == 0, 'Pure_breed'] = 1
test['Pure_breed'] = 0
test.loc[test['Breed2'] == 0, 'Pure_breed'] = 1
X=X_train
y=y_train
def rmsle(predictions, targets):
predictions = np.exp(predictions)- 1
targets = np.exp(targets)- 1
return np.sqrt(((predictions - targets)** 2 ).mean())
 def rmsle_lgb(labels, preds):
return 'rmsle', rmsle(preds, labels), False 
<train_model>
train['BadName'] = 0
train.loc[train['Name'].apply(lambda x: len(str(x)))< 3,'BadName']=1
print('Training model...')
 X_train, X_test2, y_train, y_test2 = train_test_split(X, y, test_size=0.3, random_state=42)
 lgbm_params = {'n_estimators': 1000, 'learning_rate': 0.4, 'max_depth': 10,
'num_leaves': 31, 'subsample': 0.9, 'colsample_bytree': 0.8,
'min_child_samples': 50, 'n_jobs': 4}
model = LGBMRegressor(**lgbm_params)
 model.fit(X_train, y_train,
eval_set=[(X_test2, y_test2)],
eval_metric=rmsle_lgb,
early_stopping_rounds=100,
verbose=True)
 print('Generating submission...')
 preds2 = model.predict(X_test)
<save_to_csv>
test['BadName'] = 0
test.loc[test['Name'].apply(lambda x: len(str(x)))< 3,'BadName']=1
df_test2['price2']=np.exp(preds2)- 1
df_test2['price']=(df_test2['price2']+df_test2['price1'])/2
df_test2[["test_id", "price"]].to_csv("submission.csv", index = False)
<prepare_output>
train_id = train['PetID']
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train.loc[:, 'doc_sent_mag'] = doc_sent_mag
train.loc[:, 'doc_sent_score'] = doc_sent_score
new = df_test[["test_id", "price"]].copy()
new.head()<prepare_output>
test_id = test['PetID']
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test.loc[:, 'doc_sent_mag'] = doc_sent_mag
test.loc[:, 'doc_sent_score'] = doc_sent_score
new2 = df_test2[["test_id", "price"]].copy()
new2.head()<feature_engineering>
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train.loc[:, 'vertex_x'] = vertex_xs
train.loc[:, 'vertex_y'] = vertex_ys
train.loc[:, 'bounding_confidence'] = bounding_confidences
train.loc[:, 'bounding_importance'] = bounding_importance_fracs
train.loc[:, 'dominant_blue'] = dominant_blues
train.loc[:, 'dominant_green'] = dominant_greens
train.loc[:, 'dominant_red'] = dominant_reds
train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train.loc[:, 'dominant_score'] = dominant_scores
train.loc[:, 'label_description'] = label_descriptions
train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test.loc[:, 'vertex_x'] = vertex_xs
test.loc[:, 'vertex_y'] = vertex_ys
test.loc[:, 'bounding_confidence'] = bounding_confidences
test.loc[:, 'bounding_importance'] = bounding_importance_fracs
test.loc[:, 'dominant_blue'] = dominant_blues
test.loc[:, 'dominant_green'] = dominant_greens
test.loc[:, 'dominant_red'] = dominant_reds
test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test.loc[:, 'dominant_score'] = dominant_scores
test.loc[:, 'label_description'] = label_descriptions
test.loc[:, 'label_score'] = label_scores
new['price'] = new2['price']*.6 + new['price']*.4
new.head()<save_to_csv>
target=train['AdoptionSpeed']
train=train.drop(['AdoptionSpeed'],axis=1 )
new.to_csv("lgbm_ridge_ensemble2.csv", float_format='%.4f', index=None )<define_variables>
df_train=train.drop(['Name','RescuerID','Description','PetID','label_description'],axis=1)
 df_test=test.drop(['Name','RescuerID','Description','PetID','label_description'],axis=1 )
NUM_BRANDS = 4000
NUM_CATEGORIES = 1000
NAME_MIN_DF = 10
MAX_FEATURES_ITEM_DESCRIPTION = 20000<categorify>
xg=xgb.XGBClassifier()
xg.fit(df_train,target)
 pred_xgb=xg.predict(df_test )
def rmsle(y, y0):
assert len(y)== len(y0)
 return np.sqrt(np.mean(np.power(np.log1p(y)-np.log1p(y0), 2)))
 def split_cat(text):
try: return text.split("/")
 except: return("No Label", "No Label", "No Label")
 def handle_missing_inplace(dataset):
dataset['general_cat'].fillna(value='missing', inplace=True)
 dataset['subcat_1'].fillna(value='missing', inplace=True)
 dataset['subcat_2'].fillna(value='missing', inplace=True)
 dataset['brand_name'].fillna(value='missing', inplace=True)
 dataset['item_description'].fillna(value='missing', inplace=True)
 def cutting(dataset):
pop_brand = dataset['brand_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]
dataset.loc[~dataset['brand_name'].isin(pop_brand), 'brand_name'] = 'missing'
pop_category1 = dataset['general_cat'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
pop_category2 = dataset['subcat_1'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
pop_category3 = dataset['subcat_2'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
dataset.loc[~dataset['general_cat'].isin(pop_category1), 'general_cat'] = 'missing'
dataset.loc[~dataset['subcat_1'].isin(pop_category2), 'subcat_1'] = 'missing'
dataset.loc[~dataset['subcat_2'].isin(pop_category3), 'subcat_2'] = 'missing'
def to_categorical(dataset):
dataset['general_cat'] = dataset['general_cat'].astype('category')
 dataset['subcat_1'] = dataset['subcat_1'].astype('category')
 dataset['subcat_2'] = dataset['subcat_2'].astype('category')
 dataset['item_condition_id'] = dataset['item_condition_id'].astype('category')
<load_from_csv>
rf=RandomForestClassifier()
rf.fit(df_train,target)
 pred_rf=rf.predict(df_test )
start_time = time.time()
train = pd.read_table('.. /input/train.tsv', engine='c')
 test = pd.read_table('.. /input/test_stg2.tsv', engine='c' )<prepare_x_and_y>
gb=lgb.LGBMClassifier()
gb.fit(df_train,target)
 pred_gb=gb.predict(df_test )
dftt = train[(train.price < 1.0)]
train = train.drop(train[(train.price < 1.0)].index)
 del dftt['price']
nrow_train = train.shape[0] - dftt.shape[0]
nrow_test = train.shape[0] + dftt.shape[0]
nrow_train = train.shape[0]
y = np.log1p(train["price"])
 merge = pd.concat([train, dftt, test])
 submission = test[['test_id']]<feature_engineering>
abc=AdaBoostClassifier()
abc.fit(df_train,target)
 pred_abc=abc.predict(df_test )
merge['general_cat'], merge['subcat_1'], merge['subcat_2'] = zip(*merge['category_name'].apply(lambda x: split_cat(x)))
 merge.drop('category_name', axis=1, inplace=True )<categorify>
gbc=GradientBoostingClassifier()
gbc.fit(df_train,target)
 pred_gbc=gbc.predict(df_test )
numbersList = [1, 2, 3]
strList = ['one', 'two']
numbersTuple =('ONE', 'TWO', 'THREE', 'FOUR')
 result = zip(numbersList, numbersTuple)
 resultSet = set(result)
 print(resultSet)
 result = zip(numbersList, strList, numbersTuple)
 resultSet = set(result)
 print(resultSet )<categorify>
etc=ExtraTreesClassifier()
etc.fit(df_train,target)
 pred_etc=etc.predict(df_test )
to_categorical(merge )<feature_engineering>
svc=SVC()
svc.fit(df_train,target)
 pred_svc=svc.predict(df_test )
cv = CountVectorizer(min_df=NAME_MIN_DF,ngram_range=(1, 2),
stop_words='english')
 X_name = cv.fit_transform(merge['name'] )<feature_engineering>
df=pd.DataFrame()
df['abc']=pred_abc
df['etc']=pred_etc
df['gb']=pred_gb
df['gbc']=pred_gbc
df['rf']=pred_rf
df['xgb']=pred_xgb
df['svc']=pred_svc
df.head()
del train
del test
gc.collect()
cv = CountVectorizer()
X_category1 = cv.fit_transform(merge['general_cat'])
 X_category2 = cv.fit_transform(merge['subcat_1'])
 X_category3 = cv.fit_transform(merge['subcat_2'] )<feature_engineering>
df.mode(axis=1 )
gc.collect()
tv = TfidfVectorizer(max_features=MAX_FEATURES_ITEM_DESCRIPTION,
ngram_range=(1, 3),
stop_words='english')
 X_description = tv.fit_transform(merge['item_description'] )<set_options>
submit=pd.DataFrame()
submit['PetID']=test['PetID']
submit['AdoptionSpeed']=df.mode(axis=1)[0].astype(int)
 submit.to_csv('submission.csv',index=False )