Kaggle competition: Mercari Price Suggestion Challenge
df4["year_bin"]="y0"
df4["year_bin"].loc[(df4["year"]==2011)&(df4["month"]<=3)]="y1"
df4["year_bin"].loc[(df4["year"]==2011)&(df4["month"]>3)&(df4["month"]<=6)]="y2"
df4["year_bin"].loc[(df4["year"]==2011)&(df4["month"]>6)&(df4["month"]<=9)]="y3"
df4["year_bin"].loc[(df4["year"]==2011)&(df4["month"]>9)&(df4["month"]<=12)]="y4"
df4["year_bin"].loc[(df4["year"]==2012)&(df4["month"]<=3)]="y5"
df4["year_bin"].loc[(df4["year"]==2012)&(df4["month"]>3)&(df4["month"]<=6)]="y6"
df4["year_bin"].loc[(df4["year"]==2012)&(df4["month"]>6)&(df4["month"]<=9)]="y7"
df4["year_bin"].loc[(df4["year"]==2012)&(df4["month"]>9)&(df4["month"]<=12)]="y8"
df4.groupby('year_bin')["datetime"].nunique()<import_modules>
data_train = pd.read_csv('.. /input/mercari-price-suggestion-challenge/train.tsv', delimiter='\t' )
from sklearn.cross_validation import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
from sklearn import tree<train_model>
y = np.log10(np.array(data_train['price'])+1)
 X = data_train.drop('price',axis=1)
 X_train,X_cv,Y_train,Y_cv = train_test_split(X, y, test_size=0.20, random_state=42)
 del(X, y ,data_train)
 gc.collect()
l=["hours"]
X=df4[l]
Y=df4["casual"]
dtree=DecisionTreeRegressor(max_depth=3)
 dtree.fit(X,Y )<save_to_csv>
X_train.drop('train_id', axis=1, inplace=True)
 X_cv.drop('train_id', axis=1, inplace=True )
data = export_graphviz(dtree,out_file=None,
filled=True, rounded=True,
special_characters=True)
 graph = graphviz.Source(data)
 graph<train_model>
def rmsle(y, y0):
return np.sqrt(np.mean(np.power(np.log1p(y)-np.log1p(y0), 2)) )
l=["hours"]
X=df4[l]
Y=df4["registered"]
dtree1=DecisionTreeRegressor(max_depth=4)
 dtree1.fit(X,Y )<save_to_csv>
stopwords= ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're", "you've",\
"you'll", "you'd", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', \
'she', "she's", 'her', 'hers', 'herself', 'it', "it's", 'its', 'itself', 'they', 'them', 'their',\
'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', "that'll", 'these', 'those', \
'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', \
'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', \
'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after',\
'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further',\
'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more',\
'most', 'other', 'some', 'such', 'only', 'own', 'same', 'so', 'than', 'too', 'very', \
's', 't', 'can', 'will', 'just', 'don', "don't", 'should', "should've", 'now', 'd', 'll', 'm', 'o', 're', \
've', 'y', 'ain', 'aren', "aren't", 'couldn', "couldn't", 'didn', "didn't", 'doesn', "doesn't", 'hadn',\
"hadn't", 'hasn', "hasn't", 'haven', "haven't", 'isn', "isn't", 'ma', 'mightn', "mightn't", 'mustn',\
"mustn't", 'needn', "needn't", 'shan', "shan't", 'shouldn', "shouldn't", 'wasn', "wasn't", 'weren', "weren't", \
'won', "won't", 'wouldn', "wouldn't"]
data = export_graphviz(dtree1,out_file=None,
filled=True, rounded=True,
special_characters=True)
 graph = graphviz.Source(data)
 graph<categorify>
stopwords = {x: 1 for x in stopwords.words('english')}
non_alphanums = re.compile(u'[^A-Za-z0-9]+')
 non_alphanumpunct = re.compile(u'[^A-Za-z0-9\.?!,; \(\)\[\]'"\$]+')
 RE_PUNCTUATION = '|'.join([re.escape(x)for x in string.punctuation])
 def concat_categories(x):
return set(x.values)
 def brandfinder(name, category):
for brand in brands_sorted_by_size:
if brand in name and category in brand_names_categories[brand]:
return brand
return 'Unknown'
def create_dictionary(col_name,data_frame= X_train):
dictionary = dict(zip(data_frame[col_name],data_frame.groupby(col_name)[col_name].transform('count')))
 return dictionary
def transform_col(data_frame, col_name):
dictionary = create_dictionary(col_name)
 transformed_column = []
for value in data_frame[col_name].values:
transformed_column.append(dictionary.get(value,1))
dictionary = None
del(dictionary)
 gc.collect()
return transformed_column
def clean_name(x):
if len(x):
x = non_alphanums.sub(' ', x ).split()
if len(x):
return x[0].lower()
return ''
def to_number(x):
try:
if not x.isdigit() :
return 0
x = int(x)
 if x > 100:
return 100
else:
return x
except:
return 0
def sum_numbers(desc):
if not isinstance(desc, str):
return 0
try:
return sum([to_number(s)for s in desc.split() ])
 except:
return 0
def normalize_text(text):
return u" ".join(
 [x for x in [y for y in non_alphanums.sub(' ', text ).lower().strip().split(" ")] \
if len(x)> 1 and x not in stopwords])
 def decontracted(phrase):
try:
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
except:
return 0
def cleaning_text(df):
preprocessed_item_description = []
for sentance in tqdm(df['item_description'].values):
sent = decontracted(str(sentance))
sent = sent.replace('\\r', ' ')
 sent = sent.replace('\"', ' ')
 sent = sent.replace('\
', ' ')
 sent = re.sub('[^A-Za-z0-9]+', ' ', sent)
 sent = ' '.join(e for e in sent.split() if e.lower() not in stopwords)
 preprocessed_item_description.append(sent.lower().strip())
 df = df.drop('item_description',axis=1)
 df['item_description'] = preprocessed_item_description
preprocessed_item_description = None
del(preprocessed_item_description)
 gc.collect()
return(df)
 def fill_brand(df):
brand_names_categories = dict(df[df['brand_name'] != 'missing'][['brand_name','category_name']].astype('str')\ 
.groupby('brand_name' ).agg(concat_categories ).reset_index().values.tolist())
 brands_sorted_by_size = list(sorted(filter(lambda y: len(y)>= 3, \
list(brand_names_categories.keys())) , \
key = lambda x: -len(x)))
 train_names_unknown_brands_train = df[df['brand_name'] == 'missing'][['name','category_name']].\
astype('str' ).values
train_estimated_brands_train = []
for name, category in tqdm(train_names_unknown_brands_train):
for brand in brands_sorted_by_size:
if brand in name and category in brand_names_categories[brand]:
brand_name = brand
else:
brand_name = 'missing'
train_estimated_brands_train.append(brand_name)
 df.loc[df['brand_name'] == 'missing', 'brand_name'] = train_estimated_brands_train
brand_names_categories = None
brands_sorted_by_size = None
train_names_unknown_brands_train = None
train_estimated_brands_train = None
brand_name= None
del(brand_names_categories,brands_sorted_by_size,train_names_unknown_brands_train,train_estimated_brands_train,brand_name)
 gc.collect()
return(df)
df4["daycas"]="cas0"
df4["daycas"].loc[df4["hours"]<=6.5]="cas1"
df4["daycas"].loc[(df4["hours"]>6.5)&(df4["hours"]<=7.5)]="cas2"
df4["daycas"].loc[(df4["hours"]>7.5)&(df4["hours"]<=8.5)]="cas3"
df4["daycas"].loc[(df4["hours"]>8.5)&(df4["hours"]<=9.5)]="cas4"
df4["daycas"].loc[(df4["hours"]>9.5)&(df4["hours"]<=10.5)]="cas5"
df4["daycas"].loc[(df4["hours"]>10.5)&(df4["hours"]<=19.5)]="cas6"
df4["daycas"].loc[(df4["hours"]>19.5)&(df4["hours"]<=21.5)]="cas7"
df4["daycas"].loc[df4["hours"]>21.5]="cas8"<count_unique_values>
def preprocessing(df):
print("Text_cleaning Done")
 df['brand_name'] = df['brand_name'].fillna('missing')
 df['brand_name'] = df['brand_name'].astype('category')
 print("preprocessing for brand_name Done")
 df['category_name'] = df['category_name'].fillna('missing/missing/missing')
 df['category_name'] = df['category_name'].fillna('missing/missing/missing')
 print("preprocessing for category_name Done")
 df['item_description'] = df['item_description'].fillna('missing')
 print("preprocessing for item_description Done")
 df['category_name']= df['category_name'].str.split('/')
 df['main_category'] = df['category_name'].str.get(0 ).replace('', 'missing' ).astype('category')
 df['sub_category_1'] = df['category_name'].str.get(1 ).replace('', 'missing' ).astype('category')
 df['sub_category_2'] = df['category_name'].str.get(2 ).replace('', 'missing' ).astype('category')
 print("split main category in to 3 Done")
 df['item_condition_id'] = df['item_condition_id'].astype('category')
 df['item_condition_id'] = df['item_condition_id'].cat.add_categories(['missing'] ).fillna('missing')
 print("preprocessing for item_condition_id Done")
 df['shipping'] = df['shipping'].astype('category')
 df['shipping'] = df['shipping'].cat.add_categories(['missing'] ).fillna('missing')
 print("preprocessing for shipping Done")
 df['item_description'].fillna('missing', inplace=True)
 print("preprocessing for item_description Done")
 return(df)
 def adding_new_features(df):
df['name_first'] = df['name'].apply(clean_name)
 df['name_first_count'] = transform_col(data_frame = df, col_name = 'name_first')
 df['main_cat_count'] = transform_col(data_frame = df, col_name = 'main_category')
 df['sub_cat_1_count'] = transform_col(data_frame = df, col_name = 'sub_category_1')
 df['sub_cat_2_count'] = transform_col(data_frame = df, col_name = 'sub_category_2')
 df['brand_name_count'] = transform_col(data_frame = df, col_name = 'brand_name')
 df['DescriptionLower'] = df.item_description.str.count('[a-z]')
 df['NameLower'] = df.name.str.count('[a-z]')
 df['NameUpper'] = df.name.str.count('[A-Z]')
 df['DescriptionUpper'] = df.item_description.str.count('[A-Z]')
 df['name_len'] = df['name'].apply(lambda x: len(x))
df['des_len'] = df['item_description'].apply(lambda x: len(x))
df['name_desc_len_ratio'] = df['name_len']/df['des_len']
df['desc_word_count'] = df['item_description'].apply(lambda x: len(x.split()))
df['mean_des'] = df['item_description'].apply(lambda x: 0 if len(x)== 0 else float(len(x.split())) / len(x)) * 10
df['name_word_count'] = df['name'].apply(lambda x: len(x.split()))
df['mean_name'] = df['name'].apply(lambda x: 0 if len(x)== 0 else float(len(x.split())) / len(x)) * 10
df['desc_letters_per_word'] = df['des_len'] / df['desc_word_count']
df['name_letters_per_word'] = df['name_len'] / df['name_word_count']
df['NameLowerRatio'] = df['NameLower'] / df['name_len']
df['DescriptionLowerRatio'] = df['DescriptionLower'] / df['des_len']
df['NameUpperRatio'] = df['NameUpper'] / df['name_len']
df['DescriptionUpperRatio'] = df['DescriptionUpper'] / df['des_len']
df['NamePunctCount'] = df.name.str.count(RE_PUNCTUATION)
 df['DescriptionPunctCount'] = df.item_description.str.count(RE_PUNCTUATION)
 df['NamePunctCountRatio'] = df['NamePunctCount'] / df['name_word_count']
df['DescriptionPunctCountRatio'] = df['DescriptionPunctCount'] / df['desc_word_count']
df['NameDigitCount'] = df.name.str.count('[0-9]')
 df['DescriptionDigitCount'] = df.item_description.str.count('[0-9]')
 df['NameDigitCountRatio'] = df['NameDigitCount'] / df['name_word_count']
df['DescriptionDigitCountRatio'] = df['DescriptionDigitCount']/df['desc_word_count']
df['stopword_ratio_desc'] = df['item_description'].apply(lambda x: len([w for w in x.split() if w in stopwords])) / df['desc_word_count']
df['num_sum'] = df['item_description'].apply(sum_numbers)
 df['weird_characters_desc'] = df['item_description'].str.count(non_alphanumpunct)
 df['weird_characters_name'] = df['name'].str.count(non_alphanumpunct)
 df['prices_count'] = df['item_description'].str.count('[rm]')
 df['price_in_name'] = df['item_description'].str.contains('[rm]', regex=False ).astype('int')
 df.drop('category_name', axis=1, inplace=True)
 return(df )
df4.groupby("daycas")["datetime"].nunique()<categorify>
preprocessing(X_train)
 adding_new_features(X_train )
df4["dayreg"]="reg0"
df4["dayreg"].loc[df4["hours"]<=0.5]="reg1"
df4["dayreg"].loc[(df4["hours"]>0.5)&(df4["hours"]<=1.5)]="reg2"
df4["dayreg"].loc[(df4["hours"]>1.5)&(df4["hours"]<=4.5)]="reg3"
df4["dayreg"].loc[(df4["hours"]>4.5)&(df4["hours"]<=5.5)]="reg4"
df4["dayreg"].loc[(df4["hours"]>5.5)&(df4["hours"]<=6.5)]="reg5"
df4["dayreg"].loc[(df4["hours"]>6.5)&(df4["hours"]<=8.5)]="reg6"
df4["dayreg"].loc[(df4["hours"]>8.5)&(df4["hours"]<=16.5)]="reg7"
df4["dayreg"].loc[(df4["hours"]>16.5)&(df4["hours"]<=18.5)]="reg8"
df4["dayreg"].loc[(df4["hours"]>18.5)&(df4["hours"]<=20.5)]="reg9"
df4["dayreg"].loc[(df4["hours"]>20.5)&(df4["hours"]<=21.5)]="reg10"
df4["dayreg"].loc[(df4["hours"]>21.5)&(df4["hours"]<=22.5)]="reg11"
df4["dayreg"].loc[df4["hours"]>22.5]="reg12"
df4.groupby("dayreg")["datetime"].nunique()<feature_engineering>
X_train.columns[X_train.isna().any() ].tolist()
df3["time"]=pd.to_datetime(df3["datetime"])
 df3["year"]=df3['time'].dt.year
df3["month"]=df3["time"].dt.month
df3["day"]=df3["time"].dt.dayofweek
df3.head()<categorify>
preprocessing(X_cv)
 adding_new_features(X_cv )
df3["year_bin"]="y0"
df3["year_bin"].loc[(df3["year"]==2011)&(df3["month"]<=3)]="y1"
df3["year_bin"].loc[(df3["year"]==2011)&(df3["month"]>3)&(df3["month"]<=6)]="y2"
df3["year_bin"].loc[(df3["year"]==2011)&(df3["month"]>6)&(df3["month"]<=9)]="y3"
df3["year_bin"].loc[(df3["year"]==2011)&(df3["month"]>9)&(df3["month"]<=12)]="y4"
df3["year_bin"].loc[(df3["year"]==2012)&(df3["month"]<=3)]="y5"
df3["year_bin"].loc[(df3["year"]==2012)&(df3["month"]>3)&(df3["month"]<=6)]="y6"
df3["year_bin"].loc[(df3["year"]==2012)&(df3["month"]>6)&(df3["month"]<=9)]="y7"
df3["year_bin"].loc[(df3["year"]==2012)&(df3["month"]>9)&(df3["month"]<=12)]="y8"
df3["daycas"]="cas0"
df3["daycas"].loc[df3["hours"]<=6.5]="cas1"
df3["daycas"].loc[(df3["hours"]>6.5)&(df3["hours"]<=7.5)]="cas2"
df3["daycas"].loc[(df3["hours"]>7.5)&(df3["hours"]<=8.5)]="cas3"
df3["daycas"].loc[(df3["hours"]>8.5)&(df3["hours"]<=9.5)]="cas4"
df3["daycas"].loc[(df3["hours"]>9.5)&(df3["hours"]<=10.5)]="cas5"
df3["daycas"].loc[(df3["hours"]>10.5)&(df3["hours"]<=19.5)]="cas6"
df3["daycas"].loc[(df3["hours"]>19.5)&(df3["hours"]<=21.5)]="cas7"
df3["daycas"].loc[df3["hours"]>21.5]="cas8"
df3["dayreg"]="reg0"
df3["dayreg"].loc[df3["hours"]<=0.5]="reg1"
df3["dayreg"].loc[(df3["hours"]>0.5)&(df3["hours"]<=1.5)]="reg2"
df3["dayreg"].loc[(df3["hours"]>1.5)&(df3["hours"]<=4.5)]="reg3"
df3["dayreg"].loc[(df3["hours"]>4.5)&(df3["hours"]<=5.5)]="reg4"
df3["dayreg"].loc[(df3["hours"]>5.5)&(df3["hours"]<=6.5)]="reg5"
df3["dayreg"].loc[(df3["hours"]>6.5)&(df3["hours"]<=8.5)]="reg6"
df3["dayreg"].loc[(df3["hours"]>8.5)&(df3["hours"]<=16.5)]="reg7"
df3["dayreg"].loc[(df3["hours"]>16.5)&(df3["hours"]<=18.5)]="reg8"
df3["dayreg"].loc[(df3["hours"]>18.5)&(df3["hours"]<=20.5)]="reg9"
df3["dayreg"].loc[(df3["hours"]>20.5)&(df3["hours"]<=21.5)]="reg10"
df3["dayreg"].loc[(df3["hours"]>21.5)&(df3["hours"]<=22.5)]="reg11"
df3["dayreg"].loc[df3["hours"]>22.5]="reg12"
df3.head()<drop_column>
X_cv.columns[X_cv.isna().any() ].tolist()
df6=df4.copy()
df6.drop(["datetime","season","time","count","registered","casual","logcount"],axis=1,inplace=True)
 df6.head()<feature_engineering>
total_cols = set(X_train.columns.values)
 basic_cols = {'name', 'item_condition_id', 'brand_name',
'shipping', 'item_description', 'main_category',
'sub_category_1', 'sub_category_2', 'name_first'}
numeric_cols = total_cols - basic_cols
cols_to_normalize = numeric_cols - {'price_in_name'}
text_cols = {'name', 'item_description'}
categorical_cols = {'item_condition_id','brand_name','shipping','main_category','sub_category_1','sub_category_2','name_first'}
df4["rtemp1"]=df4["temp"]+df4["atemp"]
df4["rtemp2"]=df4["temp"]-df4["atemp"]
df4["rtemp3"]=df4["temp"]*df4["atemp"]
print(df4["temp"].corr(df4["registered"]))
print(df4["rtemp1"].corr(df4["registered"]))
print(df4["rtemp2"].corr(df4["registered"]))
print(df4["rtemp3"].corr(df4["registered"]))
df3["rtemp3"]=df3["temp"]*df3["atemp"]<feature_engineering>
normalizer = Normalizer(copy=False)
 normalizer.fit(X_train[list(cols_to_normalize)] )
df6["rtemp3"]=df6["temp"]*df6["atemp"]
df6.head()<drop_column>
pickle.dump(normalizer, open("normalizer.pickle", "wb"),protocol=4)
 normalizer = None
del(normalizer)
 gc.collect()
df3.drop(["datetime","season","time"],axis=1,inplace=True )<import_modules>
def normalize_dataframe(df):
with open('normalizer.pickle',mode='rb')as model_f:
normalizer_load = pickle.load(model_f)
 df[list(cols_to_normalize)]= normalizer_load.transform(df[list(cols_to_normalize)])
 df[list({'item_condition_id','shipping'})] = df[list({'item_condition_id','shipping'})].astype('category')
 normalizer_load = None
del(normalizer_load)
 gc.collect()
return df
from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor
from sklearn.metrics import mean_squared_error<feature_engineering>
normalize_dataframe(X_train)
 normalize_dataframe(X_cv )
df7=df6.logcasual
df8=df6.logregistered<drop_column>
wb_desc = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {"hash_ngrams": 2,
"hash_ngrams_weights": [1.0, 1.0],
"hash_size": 2 ** 28,
"norm": "l2",
"tf": 1.0,
"idf": None}), procs=8)
 wb_desc.dictionary_freeze = True
df6.drop(["logcasual",'logregistered'],axis=1,inplace=True)
<drop_column>
wb_desc.fit(X_train['item_description'])
 X_description_train_wb = wb_desc.transform(X_train['item_description'])
 X_description_cv_wb = wb_desc.fit_transform(X_cv['item_description'] )
df6.drop(["atemp"],axis=1,inplace=True )<drop_column>
mask_desc = np.where(X_description_train_wb.getnnz(axis=0)> 3)[0]
df6.drop(["temp"],axis=1,inplace=True )<load_from_csv>
X_description_train_wb = X_description_train_wb[:, mask_desc]
X_description_cv_wb = X_description_cv_wb[:, mask_desc]
df6.drop(["rtemp3","month"],axis=1,inplace=True)
 d=pd.read_csv(".. /input/train.csv")
 df6["temp"]=d["temp"]
var=["holiday","workingday","weekend","hours"]
for v in var:
df6[v]=df6[v].astype("category")
<categorify>
model_desc = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=5)
 model_desc.fit(X_description_train_wb, Y_train )
dfreg=df6.copy()
dfcas=df6.copy()
dfcas['year']=dfcas.year.replace({2011:0,2012:1})
 dfreg['year']=dfcas.year.replace({2011:0,2012:1})
 dfreg.drop(["daycas"],axis=1,inplace=True)
 dureg1=pd.get_dummies(dfreg["dayreg"])
 dfreg=pd.concat([dfreg,dureg1],axis=1)
 dureg2=pd.get_dummies(dfreg["year_bin"])
 dfreg=pd.concat([dfreg,dureg2],axis=1)
 dureg3=pd.get_dummies(dfreg["day"])
 dfreg=pd.concat([dfreg,dureg3],axis=1)
 dfreg.drop(["dayreg","year_bin","day"],axis=1,inplace=True)
 dfcas.drop(["dayreg"],axis=1,inplace=True)
 ducas1=pd.get_dummies(dfcas["daycas"])
 dfcas=pd.concat([dfcas,ducas1],axis=1)
 ducas2=pd.get_dummies(dfcas["year_bin"])
 dfcas=pd.concat([dfcas,ducas2],axis=1)
 ducas3=pd.get_dummies(dfcas["day"])
 dfcas=pd.concat([dfcas,ducas3],axis=1)
 dfcas.drop(["daycas","year_bin","day"],axis=1,inplace=True )<split>
pred_train_1 = model_desc.predict(X_description_train_wb)
 pred_cv_1 = model_desc.predict(X_description_cv_wb)
 print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** pred_train_1-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** pred_cv_1-1)) )
x_train,x_test,y_train,y_test=train_test_split(dfreg,df7,random_state=42)
 rf=RandomForestRegressor(n_estimators=500)
 rf.fit(x_train,y_train)
 predictions=rf.predict(x_test)
 mean_squared_error(y_test, predictions )<train_model>
pickle.dump(mask_desc, open("mask_desc.pickle", "wb"),protocol=4)
 pickle.dump(wb_desc, open("wb_desc.pickle", "wb"),protocol=4)
 pickle.dump(model_desc, open("model_desc.pickle", "wb"),protocol=4 )
rf1=RandomForestRegressor(n_estimators=500)
 rf1.fit(dfcas,df7)
 rf2=RandomForestRegressor(n_estimators=500)
 rf2.fit(dfreg,df8 )<drop_column>
mask_desc = None
wb_desc = None
model_desc = None
del(mask_desc,wb_desc,model_desc)
 gc.collect()
df3.drop(["temp","atemp"],axis=1,inplace=True )<feature_engineering>
wb_name = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {"hash_ngrams": 2,
"hash_ngrams_weights": [1.5, 1.0],
"hash_size": 2 ** 29,
"norm": None,
"tf": 'binary',
"idf": None,
}), procs=8)
 wb_name.dictionary_freeze = True
newtest.drop(["rtemp3","month"],axis=1,inplace=True)
 f=pd.read_csv(".. /input/test.csv")
 newtest['year']=newtest.year.replace({2011:0,2012:1})
 newtest["temp"]=f["temp"]
var=["holiday","workingday","weekend","hours"]
for v in var:
newtest[v]=newtest[v].astype("category")
 newtestreg=newtest.copy()
newtestcas=newtest.copy()
newtestreg.drop(["daycas"],axis=1,inplace=True)
 newreg1=pd.get_dummies(newtestreg["dayreg"])
 newtestreg=pd.concat([newtestreg,newreg1],axis=1)
 newreg2=pd.get_dummies(newtestreg["year_bin"])
 newtestreg=pd.concat([newtestreg,newreg2],axis=1)
 newreg3=pd.get_dummies(newtestreg["day"])
 newtestreg=pd.concat([newtestreg,newreg3],axis=1)
 newtestreg.drop(["dayreg","year_bin","day"],axis=1,inplace=True)
 newtestcas.drop(["dayreg"],axis=1,inplace=True)
 newcas1=pd.get_dummies(newtestcas["daycas"])
 newtestcas=pd.concat([newtestcas,newcas1],axis=1)
 newcas2=pd.get_dummies(newtestcas["year_bin"])
 newtestcas=pd.concat([newtestcas,newcas2],axis=1)
 newcas3=pd.get_dummies(newtestcas["day"])
 newtestcas=pd.concat([newtestcas,newcas3],axis=1)
 newtestcas.drop(["daycas","year_bin","day"],axis=1,inplace=True)
 newtest.head() 
<predict_on_test>
wb_name.fit(X_train['name'])
 X_name_train_wb = wb_name.transform(X_train['name'])
 X_name_cv_wb = wb_name.fit_transform(X_cv['name'] )
predictcas=rf1.predict(newtestcas)
 predictcas=np.exp(predictcas)-1
predictreg=rf2.predict(newtestreg)
 predictreg=np.exp(predictreg)-1 
<create_dataframe>
mask_name = np.where(X_name_train_wb.getnnz(axis=0)> 3)[0]
predictcount=np.add(predictcas,predictreg)
 final=pd.DataFrame({"datetime":testdf["datetime"],"count":predictcount})
 final<save_to_csv>
X_name_train_wb = X_name_train_wb[:, mask_name]
X_name_cv_wb = X_name_cv_wb[:, mask_name]
final.to_csv("final2.csv",index=False )<load_from_csv>
model_name = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha= 5)
 model_name.fit(X_name_train_wb, Y_train)
train = pd.read_csv('/kaggle/input/cat-in-the-dat/train.csv')
 test = pd.read_csv('/kaggle/input/cat-in-the-dat/test.csv')
 sub = pd.read_csv('/kaggle/input/cat-in-the-dat/sample_submission.csv' )<define_variables>
pred_train_2 = model_name.predict(X_name_train_wb)
 pred_cv_2 = model_name.predict(X_name_cv_wb )
bin_cols = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4']
nom_cols = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
ord_cols = ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']
cyc_cols = ['day', 'month']<concatenate>
print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** pred_train_2-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** pred_cv_2-1)) )
df = train.append(test, ignore_index=True, sort=False)
 df.head(2 )<categorify>
pickle.dump(mask_name, open("mask_name.pickle", "wb"),protocol=4)
 pickle.dump(wb_name, open("wb_name.pickle", "wb"),protocol=4)
 pickle.dump(model_name, open("model_name.pickle", "wb"),protocol=4)
def date_cyc_enc(df, col, max_vals):
df[col + '_sin'] = np.sin(2 * np.pi * df[col]/max_vals)
 df[col + '_cos'] = np.cos(2 * np.pi * df[col]/max_vals)
 return df
df = date_cyc_enc(df, 'day', 7)
 df = date_cyc_enc(df, 'month', 12)
<categorify>
mask_name = None
wb_name = None
model_name = None
del(mask_name,wb_name,model_name)
 gc.collect()
df['bin_3'] = df['bin_3'].map({'T':1, 'F':0, 'Y':1, 'N':0})
 df['bin_4'] = df['bin_4'].map({'T':1, 'F':0, 'Y':1, 'N':0} )<feature_engineering>
lb_brand_name = LabelBinarizer(sparse_output=True)
 X_brand_train = lb_brand_name.fit_transform(X_train['brand_name'])
 X_brand_cv = lb_brand_name.transform(X_cv['brand_name'] )
high_card_feats = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
for col in high_card_feats:
df[f'hash_{col}'] = df[col].apply(lambda x: hash(str(x)) % 5000)
 for col in high_card_feats:
enc_nom_1 =(df.groupby(col ).size())/ len(df)
 df[f'freq_{col}'] = df[col].apply(lambda x : enc_nom_1[x] )<data_type_conversions>
lb_main_category = LabelBinarizer(sparse_output=True)
 X_main_cat_train = lb_main_category.fit_transform(X_train['main_category'])
 X_main_cat_cv = lb_main_category.transform(X_cv['main_category'] )
for col in df.columns:
if col != 'target' and df[col].dtype == 'O':
df[col] = df[col].astype('category' )<import_modules>
lb_sub_category_1 = LabelBinarizer(sparse_output=True)
 X_main_sub_cat_1_train = lb_sub_category_1.fit_transform(X_train['sub_category_1'])
 X_main_sub_cat_1_cv = lb_sub_category_1.transform(X_cv['sub_category_1'] )
from sklearn.metrics import roc_auc_score<drop_column>
lb_sub_category_2 = LabelBinarizer(sparse_output=True)
 X_main_sub_cat_2_train = lb_sub_category_2.fit_transform(X_train['sub_category_2'])
 X_main_sub_cat_2_cv = lb_sub_category_2.transform(X_cv['sub_category_2'] )
df.drop(['id'], axis=1, inplace=True)
 train_df = df[df['target'].isnull() !=True]
test_df = df[df['target'].isnull() ==True]
test_df.drop('target', axis=1, inplace=True )<split>
X_dummies_train = csr_matrix(
 pd.get_dummies(X_train[list(total_cols -(basic_cols)) ],
sparse=True ).values)
 X_dummies_train_1 = csr_matrix(
 pd.get_dummies(X_train[list({'item_condition_id', 'shipping'})],
sparse=True ).values )
X = train_df.drop(labels=['target'], axis=1)
 y = train_df['target'].values
X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=0.25, random_state=1, stratify=y )<choose_model_class>
X_dummies_cv = csr_matrix(
 pd.get_dummies(X_cv[list(total_cols -(basic_cols)) ],
sparse=True ).values)
 X_dummies_cv_1 = csr_matrix(
 pd.get_dummies(X_cv[list({'item_condition_id', 'shipping'})],
sparse=True ).values )
lgbm = LGBMClassifier(boosting_type='gbdt',
num_leaves=31,
max_depth=-1,
learning_rate=0.02,
n_estimators=2000,
subsample_for_bin=200000,
min_child_samples=20,
colsample_bytree=0.1,
random_state=0)
 lgbm.fit(X_train, y_train, eval_set=[(X_cv, y_cv)], eval_metric='auc', early_stopping_rounds=200, verbose=100)
 y_pred = lgbm.predict_proba(X_cv)[:,-1]<compute_test_metric>
sparse_merge_train = hstack(( X_name_train_wb , X_description_train_wb, X_brand_train, X_main_cat_train,
X_main_sub_cat_1_train, X_main_sub_cat_2_train,X_dummies_train,X_dummies_train_1)).tocsr()
roc_auc_score(y_cv, y_pred )<prepare_x_and_y>
sparse_merge_cv = hstack(( X_name_cv_wb,X_description_cv_wb,X_brand_cv,X_main_cat_cv,
X_main_sub_cat_1_cv,X_main_sub_cat_2_cv,X_dummies_cv,X_dummies_cv_1)).tocsr()
Xtest = test_df<train_model>
X_dummies_train = None
X_description_train_wb = None
X_name_train_wb = None
X_dummies_cv = None
X_description_cv_wb = None
X_name_cv_wb = None
del(X_dummies_train, X_description_train_wb,X_name_train_wb)
 del(X_dummies_cv, X_description_cv_wb,X_name_cv_wb)
 gc.collect()
err=[]
y_pred_tot=[]
fold=StratifiedKFold(n_splits=10, shuffle=True, random_state=1994)
 i=1
for train_index, test_index in fold.split(X,y):
X_train, X_test = X.iloc[train_index], X.iloc[test_index]
y_train, y_test = y[train_index], y[test_index]
lgbm.fit(X_train, y_train, eval_set=[(X_cv, y_cv)], eval_metric='auc', early_stopping_rounds=100, verbose=100)
 preds = lgbm.predict_proba(X_test)[:,-1]
print("ROC AUC Score: ", roc_auc_score(y_test, preds))
err.append(roc_auc_score(y_test,preds))
p = lgbm.predict_proba(Xtest)[:,-1]
print(f'-------------------- Fold {i} completed !!! ------------------')
 i=i+1
y_pred_tot.append(p )<feature_engineering>
pickle.dump(lb_brand_name, open("lb_brand_name.pickle", "wb"),protocol=4)
 pickle.dump(lb_main_category, open("lb_main_category.pickle", "wb"),protocol=4)
 pickle.dump(lb_sub_category_1, open("lb_sub_category_1.pickle", "wb"),protocol=4)
 pickle.dump(lb_sub_category_2, open("lb_sub_category_2.pickle", "wb"),protocol=4 )
err_avg = np.mean(err,0)
 err_avg<prepare_output>
lb_brand_name = None
lb_main_category = None
lb_sub_category_1 = None
lb_sub_category_2 = None
del(lb_brand_name,lb_main_category,lb_sub_category_1,lb_sub_category_2)
 gc.collect()
y_pred = np.mean(y_pred_tot,0)
 y_pred<prepare_output>
model_FM_FTRL = FM_FTRL(alpha=0.035, beta=0.001, L1=0.00001, L2=0.15, D=sparse_merge_train.shape[1],
alpha_fm=0.05, L2_fm=0.0, init_fm=0.01,
D_fm=100, e_noise=0, iters=1, inv_link="identity", threads=4)
 model_FM_FTRL.fit(sparse_merge_train, Y_train )
sub['target'] = y_pred<save_to_csv>
pred_train_3 = model_FM_FTRL.predict(sparse_merge_train)
 pred_cv_3 = model_FM_FTRL.predict(sparse_merge_cv )
sub.to_csv('10fold_lgbm.csv', index=False )<load_from_csv>
print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** pred_train_3-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** pred_cv_3-1)) )
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv', index_col='id')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv', index_col='id' )<drop_column>
pickle.dump(model_FM_FTRL, open("model_FM_FTRL.pickle", "wb"),protocol=4)
 model_FM_FTRL = None
del(model_FM_FTRL)
 gc.collect()
train = train.drop('bin_0', axis=1)
 test = test.drop('bin_0', axis=1 )<categorify>
fselect = SelectKBest(f_regression, k=48000)
 train_kbest_features = fselect.fit_transform(sparse_merge_train, Y_train)
 cv_kbest_features = fselect.transform(sparse_merge_cv )
map_bin_3_4 = {'T': 1, 'F': 0, 'Y': 1, 'N': 0}
train['bin_3'] = train['bin_3'].map(map_bin_3_4)
 test['bin_3'] = test['bin_3'].map(map_bin_3_4)
 train['bin_4'] = train['bin_4'].map(map_bin_3_4)
 test['bin_4'] = test['bin_4'].map(map_bin_3_4 )<drop_column>
pickle.dump(fselect, open("fselect.pickle", "wb"),protocol=4)
 fselect = None
del(fselect)
 gc.collect()
train['ord_5_1'] = train['ord_5'].str[0]
train['ord_5_2'] = train['ord_5'].str[1]
train = train.drop('ord_5', axis=1)
 test['ord_5_1'] = test['ord_5'].str[0]
test['ord_5_2'] = test['ord_5'].str[1]
test = test.drop('ord_5', axis=1 )<drop_column>
sparse_merge_train= None
sparse_merge_cv = None
del(sparse_merge_train,sparse_merge_cv)
 gc.collect()
train = train.drop('ord_5_2', axis=1)
 test = test.drop('ord_5_2', axis=1 )<feature_engineering>
tfidf_desc = TfidfVectorizer(max_features=500000,
ngram_range=(1, 3),
stop_words=None)
 X_desc_train_tfidf = tfidf_desc.fit_transform(X_train['item_description'] )
ord_1_values = ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster']
map_ord_1 = lambda x: ord_1_values.index(x)
 train['ord_1'] = train['ord_1'].apply(map_ord_1)
 test['ord_1'] = test['ord_1'].apply(map_ord_1 )<feature_engineering>
X_desc_cv_tfidf = tfidf_desc.transform(X_cv['item_description'] )
ord_2_values = ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot']
map_ord_2 = lambda x: ord_2_values.index(x)
 train['ord_2'] = train['ord_2'].apply(map_ord_2)
 test['ord_2'] = test['ord_2'].apply(map_ord_2 )<feature_engineering>
pickle.dump(tfidf_desc, open("tfidf_desc.pickle", "wb"),protocol=4)
 tfidf_desc = None
del(tfidf_desc)
 gc.collect()
map_to_ascii_index = lambda x: string.ascii_letters.index(x)
 for column in ['ord_3', 'ord_4', 'ord_5_1']:
train[column] = train[column].apply(map_to_ascii_index)
 test[column] = test[column].apply(map_to_ascii_index )<feature_engineering>
tfidf_name = TfidfVectorizer(max_features=250000,
ngram_range=(1, 3),
stop_words=None)
 X_name_train_tfidf = tfidf_name.fit_transform(X_train['name'] )
columns_to_test = ['nom_7', 'nom_8', 'nom_9']
replace_xor = lambda x: 'xor' if x in xor_values else x
for column in columns_to_test:
xor_values = set(train[column].unique())^ set(test[column].unique())
 if xor_values:
print('Column', column, 'has', len(xor_values), 'XOR values')
 train[column] = train[column].apply(replace_xor)
 test[column] = test[column].apply(replace_xor)
 else:
print('Column', column, 'has no XOR values' )<count_values>
X_name_cv_tfidf =tfidf_name.transform(X_cv['name'] )
def get_infrequent_values(data, column, threshold):
value_counts = data[column].value_counts()
return list(value_counts[value_counts < threshold].index )<count_unique_values>
pickle.dump(tfidf_name, open("tfidf_name.pickle", "wb"),protocol=4)
 tfidf_name = None
del(tfidf_name)
 gc.collect()
for column in train.columns:
n = len(get_infrequent_values(train, column, 3))
if n > 0:
print('Column', column, 'has', n, 'unique infrequent value(s)' )<categorify>
sparse_merge_train_1 = hstack(( X_name_train_tfidf , X_desc_train_tfidf, X_brand_train, X_main_cat_train,
X_main_sub_cat_1_train, X_main_sub_cat_2_train,X_dummies_train_1)).tocsr()
thresholds = {
'nom_9': 3
}
for column in thresholds.keys() :
values_to_replace = get_infrequent_values(train, column, thresholds[column])
 replace = lambda x: 'value' if x in values_to_replace else x
train[column] = train[column].transform(replace)
 test[column] = test[column].transform(replace )<count_values>
X_dummies_train_1 = None
X_brand_train = None
X_main_cat_train = None
X_main_sub_cat_1_train = None
X_main_sub_cat_2_train = None
X_name_train_tfidf = None
X_desc_train_tfidf = None
del(X_dummies_train_1, X_brand_train, X_main_cat_train,\
X_main_sub_cat_1_train, X_main_sub_cat_2_train, X_name_train_tfidf,X_desc_train_tfidf)
 gc.collect()
for column in thresholds.keys() :
print(column, ':', train[column].value_counts() ['value'], 'values replaced in train dataset' )<define_variables>
sparse_merge_cv_1 = hstack(( X_name_cv_tfidf , X_desc_cv_tfidf, X_brand_cv, X_main_cat_cv,
X_main_sub_cat_1_cv, X_main_sub_cat_2_cv,X_dummies_cv_1)).tocsr()
all_features = test.columns
selected_features_for_thermometer = ['ord_1', 'ord_4', 'day', 'month', 'ord_5_1']
selected_features_for_both = ['ord_2', 'ord_3']<prepare_x_and_y>
X_dummies_cv_1 = None
X_brand_cv = None
X_main_cat_cv = None
X_main_sub_cat_1_cv = None
X_main_sub_cat_2_cv = None
X_name_cv_tfidf = None
X_desc_cv_tfidf = None
del(X_dummies_cv_1, X_brand_cv, X_main_cat_cv,\
X_main_sub_cat_1_cv, X_main_sub_cat_2_cv, X_name_cv_tfidf,X_desc_cv_tfidf)
 gc.collect()
y_train = train['target'].copy()
x_train = train.drop('target', axis=1)
 del train
x_test = test.copy()
del test<train_model>
model_Ridge_set_2 = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=5)
 model_Ridge_set_2.fit(sparse_merge_train_1, Y_train )
class ThermometerEncoder(BaseEstimator, TransformerMixin):
def __init__(self, cols=None, drop_invariant=True):
self.cols = cols
self.drop_invariant = drop_invariant
def get_params(self, deep=True):
return {'cols': self.cols, 'drop_invariant': self.drop_invariant}
def set_params(self, **parameters):
for parameter, value in parameters.items() :
setattr(self, parameter, value)
 return self
def fit(self, X, y=None):
self.bars = {}
for c in self.cols:
k = np.arange(X[c].max() + 1)
 self.bars[c] =(k[:-1] < k.reshape(-1, 1)).astype(int)
 return self
def transform(self, X, y=None):
out = pd.DataFrame(index=X.index)
 for c in self.cols:
out = out.join(self.transform_one(X, c))
if self.drop_invariant:
columns_to_drop = []
for c in out.columns:
if len(out[c].unique())== 1:
columns_to_drop.append(c)
 out = out.drop(columns_to_drop, axis=1)
 return out
def transform_one(self, X, c):
bars = self.bars[c]
out = pd.DataFrame(index=X.index, data=bars[X[c]])
 out.columns = [c + '_' + str(k)for k in range(bars.shape[1])]
return out<concatenate>
pred_train_4 = model_Ridge_set_2.predict(sparse_merge_train_1)
 pred_cv_4 = model_Ridge_set_2.predict(sparse_merge_cv_1)
 print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** pred_train_4-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** pred_cv_4-1)) )
train_part = len(x_train)
 traintest = pd.concat([x_train, x_test] )<define_variables>
pickle.dump(model_Ridge_set_2, open("model_Ridge_set_2.pickle", "wb"),protocol=4)
 model_Ridge_set_2 = None
del(model_Ridge_set_2)
 gc.collect()
ohe_features = set(all_features)- set(selected_features_for_thermometer )<categorify>
model_MNB_set_2 = MultinomialNB(alpha=1.0, fit_prior=True)
 model_MNB_set_2.fit(sparse_merge_train_1, Y_train >= 4 )
ohe_traintest = pd.get_dummies(traintest[ohe_features], columns=ohe_features, drop_first=True, sparse=True ).sparse.to_coo().tocsr()<categorify>
pred_train_5 = model_MNB_set_2.predict(sparse_merge_train_1)
 pred_cv_5 = model_MNB_set_2.predict(sparse_merge_cv_1)
 print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** pred_train_5-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** pred_cv_5-1)) )
thermometer = ThermometerEncoder(selected_features_for_thermometer + selected_features_for_both)
 thermometer_traintest = thermometer.fit_transform(traintest )<concatenate>
pickle.dump(model_MNB_set_2, open("model_MNB_set_2.pickle", "wb"),protocol=4)
 model_MNB_set_2 = None
sparse_merge_train_1 = None
sparse_merge_cv_1 = None
del(model_MNB_set_2)
 del(sparse_merge_train_1,sparse_merge_cv_1)
 gc.collect()
final_traintest = scipy.sparse.hstack([ohe_traintest, thermometer_traintest] ).tocsr()<prepare_x_and_y>
f_cats = ['brand_name', 'main_category', 'sub_category_1', 'sub_category_2', 'name_first']
final_x_train = final_traintest[:train_part]
final_x_test = final_traintest[train_part:]<drop_column>
targetencoder = TargetEncoder(min_samples_leaf=100, smoothing=10,cols=f_cats,return_df=False )
del x_train
del x_test<import_modules>
targetencoder.fit(X_train[f_cats],Y_train )
from sklearn.model_selection import KFold
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_auc_score
from cross_validation_framework import *<train_on_grid>
X_train_target_encode = targetencoder.transform(X_train[f_cats] )
logit_param_grid = {
'C': [0.100, 0.150, 0.120, 0.125, 0.130, 0.135, 0.140, 0.145, 0.150]
}
logit_grid = GridSearchCV(LogisticRegression(solver='lbfgs'), logit_param_grid,
scoring='roc_auc', cv=5, n_jobs=-1, verbose=0)
 logit_grid.fit(final_x_train, y_train)
 best_C = logit_grid.best_params_['C']
print('Best C:', best_C )<find_best_model_class>
X_cv_target_encode = targetencoder.transform(X_cv[f_cats] )
logit = LogisticRegression(C=best_C, solver='lbfgs', max_iter=10000)
 cv = KFold(n_splits=10, random_state=42)
 _, trained_estimators = fit(ScikitLearnPredictProbaEstimator(logit), roc_auc_score, final_x_train, y_train, cv)
 y = predict(trained_estimators, final_x_test )<train_model>
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
logit = LogisticRegression(C=0.123456789, solver="lbfgs", max_iter=5000)
 logit.fit(final_x_train, y_train)
 y_full_train = logit.predict_proba(final_x_test)[:, 1]<save_to_csv>
X_train_target_encode[:,0] = add_noise(X_train_target_encode[:,0],noise_level=0.01)
 X_train_target_encode[:,1] = add_noise(X_train_target_encode[:,1],noise_level=0.01)
 X_train_target_encode[:,2] = add_noise(X_train_target_encode[:,2],noise_level=0.01)
 X_train_target_encode[:,3] = add_noise(X_train_target_encode[:,3],noise_level=0.01)
 X_train_target_encode[:,4] = add_noise(X_train_target_encode[:,4],noise_level=0.01 )
submission = pd.read_csv('.. /input/cat-in-the-dat/sample_submission.csv', index_col='id')
 submission['target'] = y
submission.to_csv('logit.csv' )<save_to_csv>
pickle.dump(targetencoder, open("targetencoder.pickle", "wb"),protocol=4)
 targetencoder = None
del(targetencoder)
 gc.collect()
submission['target'] = y_full_train
submission.to_csv('logit-full-train.csv' )<import_modules>
train_features = hstack(( pred_train_1.reshape(-1,1),\
pred_train_2.reshape(-1,1),\
pred_train_3.reshape(-1,1),\
pred_train_4.reshape(-1,1),\
pred_train_5.reshape(-1,1),\
X_train_target_encode,\
train_kbest_features)).tocsr()
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import gmean
from sklearn.model_selection import cross_val_predict, cross_val_score, StratifiedKFold, StratifiedShuffleSplit
from sklearn.metrics import roc_auc_score
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline, make_union
from sklearn.preprocessing import OneHotEncoder, QuantileTransformer, MinMaxScaler, StandardScaler<define_variables>
cv_features = hstack(( pred_cv_1.reshape(-1,1),\
pred_cv_2.reshape(-1,1),\
pred_cv_3.reshape(-1,1),\
pred_cv_4.reshape(-1,1),\
pred_cv_5.reshape(-1,1),\
X_cv_target_encode,\
cv_kbest_features)).tocsr()
ROOT = '/kaggle/input/cat-in-the-dat/'
class FieldNames:
idx = 'id'
bin_0 = 'bin_0'
bin_1 = 'bin_1'
bin_2 = 'bin_2'
bin_3 = 'bin_3'
bin_4 = 'bin_4'
nom_0 = 'nom_0'
nom_1 = 'nom_1'
nom_2 = 'nom_2'
nom_3 = 'nom_3'
nom_4 = 'nom_4'
nom_5 = 'nom_5'
nom_6 = 'nom_6'
nom_7 = 'nom_7'
nom_8 = 'nom_8'
nom_9 = 'nom_9'
ord_0 = 'ord_0'
ord_1 = 'ord_1'
ord_2 = 'ord_2'
ord_3 = 'ord_3'
ord_4 = 'ord_4'
ord_5 = 'ord_5'
ord_5a = 'ord_5a'
ord_5b = 'ord_5b'
day = 'day'
month = 'month'
target = 'target'
class FileNames:
train = 'train.csv'
test = 'test.csv'
train_v1 = 'train_v1.ftr'
test_v1 = 'test_v1.ftr'
sub = 'submission.csv'
class ColumnGroup:
binary = [FieldNames.bin_0, FieldNames.bin_1, FieldNames.bin_2,
FieldNames.bin_3, FieldNames.bin_4]
ordinal = [FieldNames.ord_0, FieldNames.ord_1, FieldNames.ord_2,
FieldNames.ord_3, FieldNames.ord_4, FieldNames.ord_5,
]
nominal = [FieldNames.nom_0, FieldNames.nom_1, FieldNames.nom_2,
FieldNames.nom_3, FieldNames.nom_4, FieldNames.nom_5,
FieldNames.nom_6, FieldNames.nom_7, FieldNames.nom_8,
FieldNames.nom_9]<categorify>
pickle.dump(train_features, open("train_features.pickle", "wb"),protocol=4)
 pickle.dump(cv_features, open("cv_features.pickle", "wb"),protocol=4 )
def _convert_to_2d_array(X):
X = np.array(X)
 if X.ndim == 1:
return X.reshape(-1, 1)
 return X
class BaseTransformer(BaseEstimator, TransformerMixin):

 def fit(self, X, y=None):

 return self
@abstractmethod
def _transform(self, X):
pass
def transform(self, X, y=None):

 return self._transform(X)
 class ArrayTransformer(BaseTransformer):

 def transform(self, X):

 Xt = self._transform(X)
 return _convert_to_2d_array(Xt)
 class SelectCols(BaseTransformer):

 def __init__(self, cols):

 self.cols = cols
def _transform(self, X, y=None):
return X[self.cols]
class LabelEncoderWithThresh(ArrayTransformer):

 def __init__(self, threshold=0):
self.threshold = threshold
self.label2int = []
def fit(self, X, y=None):
self.label2int = []
X = np.asarray(X)
 m = X.shape[1]
if not isinstance(self.threshold, list):
thresholds = [self.threshold]*m
else:
thresholds = self.threshold
for j in range(m):
cnts = Counter(X[:, j])
 self.label2int.append({lbl: i+1 for i,(lbl, cnt)in enumerate(cnts.items())if cnt >
thresholds[j]})
 return self
def _transform(self, X):
X = np.asarray(X)
 Xt = np.zeros_like(X)
 for i in range(len(self.label2int)) :
Xt[:, i] = [self.label2int[i].get(val, 0)for val in X[:, i]]
return Xt 
<count_unique_values>
pred_train_1 = None
pred_train_2 = None
pred_train_3 = None
pred_train_4 = None
pred_train_5 = None
X_train_target_encode = None
train_kbest_features = None
pred_cv_1 = None
pred_cv_2 = None
pred_cv_3 = None
pred_cv_4 = None
pred_cv_5 = None
X_cv_target_encode = None
cv_kbest_features = None
del(pred_train_1,\
pred_train_2,\
pred_train_3,\
pred_train_4,\
pred_train_5,\
X_train_target_encode,\
train_kbest_features)
 del(pred_cv_1,\
pred_cv_2,\
pred_cv_3,\
pred_cv_4,\
pred_cv_5,\
X_cv_target_encode,\
cv_kbest_features)
 gc.collect()
def map_ordinals(df):
ord_1_map = {'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3, 'Grandmaster': 4.0}
ord_2_map = {'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5.3}
ord_3_map = {char: i for i, char in enumerate(sorted(df.ord_3.unique())) }
ord_4_map = {char: i for i, char in enumerate(sorted(df.ord_4.unique())) }
ord_5_map = {char: i for i, char in enumerate(sorted(df.ord_5.unique())) }
bin_4_map = {'N': 0, 'Y': 1}
bin_3_map = {'F': 0, 'T': 1}
df[FieldNames.ord_1] = df[FieldNames.ord_1].map(ord_1_map)
 df[FieldNames.ord_2] = df[FieldNames.ord_2].map(ord_2_map)
 df[FieldNames.ord_3] = df[FieldNames.ord_3].map(ord_3_map)
 df[FieldNames.ord_4] = df[FieldNames.ord_4].map(ord_4_map)
 df[FieldNames.ord_5a] = df[FieldNames.ord_5].str[0]
df[FieldNames.ord_5b] = df[FieldNames.ord_5].str[1]
ord_5a_map = {char: i for i, char in enumerate(sorted(df.ord_5a.unique())) }
ord_5b_map = {char: i for i, char in enumerate(sorted(df.ord_5b.unique())) }
df[FieldNames.ord_5a] = df[FieldNames.ord_5a].map(ord_5a_map)
 df[FieldNames.ord_5b] = df[FieldNames.ord_5b].map(ord_5b_map)
 df[FieldNames.ord_5] = df[FieldNames.ord_5].map(ord_5_map)
 df[FieldNames.bin_3] = df[FieldNames.bin_3].map(bin_3_map)
 df[FieldNames.bin_4] = df[FieldNames.bin_4].map(bin_4_map)
 return df<load_from_csv>
d_train = lgb.Dataset(train_features, label=Y_train)
 d_valid = lgb.Dataset(cv_features, label=Y_cv)
 watchlist = [d_train, d_valid]
tr = pd.read_csv(ROOT + FileNames.train)
 te = pd.read_csv(ROOT + FileNames.test)
 tr = map_ordinals(tr)
 te = map_ordinals(te)
 print('Mapping binary and ordinal columns done.')
 y = tr.target.values.astype(int)
 cvlist = list(StratifiedShuffleSplit(n_splits=5, test_size=0.05, random_state=123457869 ).split(tr, y))<categorify>
params = {
'colsample_bytree': 0.42799939792816927,
'max_depth': 8,
'min_child_samples': 370,
'min_child_weight': 0.01,
'num_leaves': 29,
'reg_lambda': 5,
'subsample': 0.6739316550896339,
'learning_rate':0.1,
'reg_alpha' :0.5,
'boosting_type': 'gbdt',
'objective' : 'regression',
'metric' : 'RMSE',
'verbosity': -1,
'lambda_l1': 10,
'lambda_l2': 10
}
qtr_map = {1:1, 2:1, 3:1, 4:2, 5:2, 6:2, 7:3, 8:3, 9:3, 10:4, 11:4, 12:4}
tr['qtr'] = tr['month'].map(qtr_map)
 te['qtr'] = te['month'].map(qtr_map )<categorify>
model_lgb_final = lgb.train(params,
train_set=d_train,
num_boost_round=3000,
valid_sets=watchlist,
verbose_eval=200,early_stopping_rounds=100)
day_map = {4:0, 5:1, 3:1, 6:2, 2:2, 7:3, 1:3}
tr['day2'] = tr['day'].map(day_map)
 te['day2'] = te['day'].map(day_map )<categorify>
pred_train_6 = model_lgb_final.predict(train_features)
 pred_cv_6 = model_lgb_final.predict(cv_features )
mm_map = {1:0, 2:0, 3:1, 4:1, 5:2, 6:2, 7:3, 8:3, 9:4, 10:4, 11:5, 12:5}
tr['month2'] =(tr['month'] >= 6 ).astype(int)
 te['month2'] =(te['month'] >= 6 ).astype(int )<define_variables>
print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** pred_train_6-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** pred_cv_6-1)) )
bin_cols = ColumnGroup.binary
ord_cols = ColumnGroup.ordinal
nom_cols = ColumnGroup.nominal
cyc_cols = ['day', 'month']<set_options>
pickle.dump(model_lgb_final, open("model_lgb_final.pickle", "wb"),protocol=4)
 model_lgb_final = None
del(model_lgb_final)
 gc.collect()
pd.set_option('display.max_columns', 200)
 tr.loc[(tr.ord_4 == 11)&(tr.ord_5 == 11)]<import_modules>
train_features= None
cv_features = None
d_train = None
d_valid = None
watchlist = None
del(train_features,cv_features,d_train,d_valid,watchlist)
 gc.collect()
from sklearn.decomposition import TruncatedSVD
import lightgbm as lgb<categorify>
X_train = None
X_cv = None
Y_train = None
Y_test = None
del(X_train,X_cv,Y_train,Y_cv)
 gc.collect()
feat_pipe2 = make_union(
 make_pipeline(SelectCols(nom_cols+['month', 'day2']),
LabelEncoderWithThresh(3),
OneHotEncoder() , 
),
make_pipeline(SelectCols(bin_cols)) ,
make_pipeline(SelectCols(ord_cols + ['qtr', 'ord_5a']),
MinMaxScaler(( 0, 1))), 
)
 feat_pipe1 = make_union(
 make_pipeline(SelectCols(nom_cols+['month', 'day']),
LabelEncoderWithThresh(1),
OneHotEncoder() , 
),
make_pipeline(SelectCols(bin_cols)) ,
make_pipeline(SelectCols(ord_cols + ['ord_5a']),
MinMaxScaler(( 0, 1))), 
)
 model = LogisticRegression(C=0.11, solver='lbfgs', random_state=12345786, max_iter=5000, tol=1e-8, fit_intercept=True)
<find_best_model_class>
test_id = []
prediction = []
def cross_val_pred(pipe, model, tr, y, te, cvlist):
scores = []
val_preds = []
y_trues = []
test_preds = []
pipe.fit(pd.concat([tr, te]))
for trix, vlix in cvlist:
xtr, ytr = tr.iloc[trix], y[trix]
xvl, yvl = tr.iloc[vlix], y[vlix]
pipe.fit(xtr, ytr)
 xtr = pipe.transform(xtr)
 xvl = pipe.transform(xvl)
 print(xtr.shape)
 model.fit(xtr, ytr)
 pvl = model.predict_proba(xvl)[:, 1]
val_preds.extend(pvl)
 y_trues.extend(yvl)
 score = roc_auc_score(yvl, pvl)
 print("Score ", score)
 scores.append(score)
 print("Avg score ", np.mean(scores), np.std(scores))
print("Overall score ", roc_auc_score(y_trues, val_preds))
xtr = pipe.transform(tr)
 xte = pipe.transform(te)
 test_preds = model.fit(xtr, y ).predict_proba(xte)[:, 1]
return val_preds, y_trues, test_preds<compute_train_metric>
def predict_final(df):
test_id.extend(list(df['test_id']))
df.drop('test_id', axis=1, inplace=True)
 preprocessing(df)
 adding_new_features(df)
 normalize_dataframe(df)
 with open('mask_desc.pickle',mode='rb')as model_f:
mask_desc = pickle.load(model_f)
 with open('wb_desc.pickle',mode='rb')as model_f:
wb_desc= pickle.load(model_f)
 with open('model_desc.pickle',mode='rb')as model_f:
model_desc = pickle.load(model_f)
 X_description_test_wb = wb_desc.transform(df['item_description'])
 X_description_test_wb = X_description_test_wb[:, mask_desc]
print(X_description_test_wb.shape)
 pred_test_1 = model_desc.predict(X_description_test_wb)
 mask_desc = None
wb_desc = None
model_desc = None
del(mask_desc,wb_desc,model_desc)
 gc.collect()
with open('mask_name.pickle',mode='rb')as model_f:
mask_name = pickle.load(model_f)
 with open('wb_name.pickle',mode='rb')as model_f:
wb_name= pickle.load(model_f)
 with open('model_name.pickle',mode='rb')as model_f:
model_name = pickle.load(model_f)
 X_name_test_wb = wb_name.transform(df['name'])
 X_name_test_wb = X_name_test_wb[:, mask_name]
print(X_name_test_wb.shape)
 pred_test_2 = model_name.predict(X_name_test_wb)
 mask_name = None
wb_name = None
model_name = None
del(mask_name,wb_name,model_name)
 gc.collect()
with open('lb_brand_name.pickle',mode='rb')as model_f:
lb_brand_name = pickle.load(model_f)
 with open('lb_main_category.pickle',mode='rb')as model_f:
lb_main_category = pickle.load(model_f)
 with open('lb_sub_category_1.pickle',mode='rb')as model_f:
lb_sub_category_1 = pickle.load(model_f)
 with open('lb_sub_category_2.pickle',mode='rb')as model_f:
lb_sub_category_2 = pickle.load(model_f)
 X_brand_test = lb_brand_name.transform(df['brand_name'])
 X_main_cat_test = lb_main_category.transform(df['main_category'])
 X_main_sub_cat_1_test = lb_sub_category_1.transform(df['sub_category_1'])
 X_main_sub_cat_2_test = lb_sub_category_2.transform(df['sub_category_2'])
 X_dummies_test = csr_matrix(
 pd.get_dummies(df[list(total_cols -(basic_cols)) ],
sparse=True ).values)
 X_dummies_test_1 = csr_matrix(
 pd.get_dummies(df[list({'item_condition_id', 'shipping'})],
sparse=True ).values)
 sparse_merge_test = hstack(( X_name_test_wb , X_description_test_wb, X_brand_test, X_main_cat_test,
X_main_sub_cat_1_test, X_main_sub_cat_2_test,X_dummies_test,X_dummies_test_1)).tocsr()
X_dummies_test = None
X_description_test_wb = None
X_name_test_wb = None
del(X_dummies_test, X_description_test_wb,X_name_test_wb)
 gc.collect()
print(sparse_merge_test.shape)
 with open('model_FM_FTRL.pickle',mode='rb')as model_f:
model_FM_FTRL = pickle.load(model_f)
 pred_test_3 = model_FM_FTRL.predict(sparse_merge_test)
 model_FM_FTRL = None
del(model_FM_FTRL)
 gc.collect()
with open('fselect.pickle',mode='rb')as model_f:
fselect = pickle.load(model_f)
 test_kbest_features = fselect.transform(sparse_merge_test)
 fselect = None
print(test_kbest_features.shape)
 del(fselect)
 gc.collect()
sparse_merge_train= None
sparse_merge_cv = None
del(sparse_merge_train,sparse_merge_cv)
 gc.collect()
with open('tfidf_desc.pickle',mode='rb')as model_f:
tfidf_desc = pickle.load(model_f)
 X_desc_test_tfidf = tfidf_desc.transform(df['item_description'])
 tfidf_desc = None
del(tfidf_desc)
 gc.collect()
with open('tfidf_name.pickle',mode='rb')as model_f:
tfidf_name = pickle.load(model_f)
 X_name_test_tfidf = tfidf_name.transform(df['name'])
 tfidf_name = None
del(tfidf_name)
 gc.collect()
sparse_merge_test_1 = hstack(( X_name_test_tfidf , X_desc_test_tfidf, X_brand_test, X_main_cat_test,
X_main_sub_cat_1_test, X_main_sub_cat_2_test,X_dummies_test_1)).tocsr()
X_dummies_test_1 = None
X_brand_test = None
X_main_cat_test = None
X_main_sub_cat_1_test = None
X_main_sub_cat_2_test = None
X_name_test_tfidf = None
X_desc_test_tfidf = None
del(X_dummies_test_1, X_brand_test, X_main_cat_test,\
X_main_sub_cat_1_test, X_main_sub_cat_2_test, X_name_test_tfidf,X_desc_test_tfidf)
 gc.collect()
with open('model_Ridge_set_2.pickle',mode='rb')as model_f:
model_Ridge_set_2 = pickle.load(model_f)
 pred_test_4 = model_Ridge_set_2.predict(sparse_merge_test_1)
 model_Ridge_set_2 = None
del(model_Ridge_set_2)
 gc.collect()
with open('model_MNB_set_2.pickle',mode='rb')as model_f:
model_MNB_set_2 = pickle.load(model_f)
 pred_test_5 = model_MNB_set_2.predict(sparse_merge_test_1)
 model_MNB_set_2 = None
sparse_merge_train_1 = None
sparse_merge_cv_1 = None
del(model_MNB_set_2)
 del(sparse_merge_train_1,sparse_merge_cv_1)
 gc.collect()
with open('targetencoder.pickle',mode='rb')as model_f:
targetencoder = pickle.load(model_f)
 f_cats = ['brand_name', 'main_category', 'sub_category_1', 'sub_category_2', 'name_first']
X_test_target_encode = targetencoder.transform(df[f_cats])
 targetencoder = None
del(targetencoder)
 gc.collect()
test_features = hstack(( pred_test_1.reshape(-1,1),\
pred_test_2.reshape(-1,1),\
pred_test_3.reshape(-1,1),\
pred_test_4.reshape(-1,1),\
pred_test_5.reshape(-1,1),\
X_test_target_encode,\
test_kbest_features)).tocsr()
pred_test_1 = None
pred_test_2 = None
pred_test_3 = None
pred_test_4 = None
pred_test_5 = None
X_test_target_encode = None
test_kbest_features = None
del(pred_test_1,\
pred_test_2,\
pred_test_3,\
pred_test_4,\
pred_test_5,\
X_test_target_encode,\
test_kbest_features)
 gc.collect()
with open('model_lgb_final.pickle',mode='rb')as model_f:
model_lgb_final = pickle.load(model_f)
 prediction.extend(list(model_lgb_final.predict(test_features)))
 train_features = None
cv_features = None
d_train = None
d_valid = None
watchlist = None
del(train_features,cv_features,d_train,d_valid,watchlist)
 gc.collect()
val_preds1, y_trues, test_preds1 = cross_val_pred(feat_pipe2, model, tr, y, te, cvlist)
<compute_test_metric>
chunksize = 10 ** 6
for chunk in pd.read_csv('.. /input/mercari-price-suggestion-challenge/test_stg2.tsv', delimiter='\t',chunksize=chunksize):
predict_final(chunk )
<save_to_csv><EOS> 
submission = pd.DataFrame()
submission['test_id'] = np.asarray(test_id)
 submission['price'] =(10 ** np.asarray(prediction)- 1)
 submission.to_csv('stacked_submission_1.csv', index=False )