File size: 4,901 Bytes
b05ab4f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 |
import numpy as np
import pandas as pd
import json
from Quin.Core import ModelOptimization
EPSILON = 1e-5
class FeatureEngineer(ModelOptimization):
def apply(self, df, k, condition):
df[k] = df['features'].apply(condition)
df[k] = df[k].astype(np.int8)
def fit(self, X, y=None, **fit_params):
return self
def transform(self, X, y=None):
df = X.copy()
df.features = df.features.apply(lambda x: ' '.join([y.replace(' ', '_') for y in x]))
df.features = df.features.apply(lambda x: x.lower())
df.features = df.features.apply(lambda x: x.replace('-', '_'))
for k, condition in (('dishwasher', lambda x: 'dishwasher' in x),
('doorman', lambda x: 'doorman' in x or 'concierge' in x),
('pets', lambda x: "pets" in x or "pet" in x or "dog" in x or "cats" in x and "no_pets" not in x),
('air_conditioning', lambda x: 'air_conditioning' in x or 'central' in x),
('parking', lambda x: 'parking' in x),
('balcony', lambda x: 'balcony' in x or 'deck' in x or 'terrace' in x or 'patio' in x),
('bike', lambda x: 'bike' in x),
('storage', lambda x: 'storage' in x),
('outdoor', lambda x: 'outdoor' in x or 'courtyard' in x or 'garden' in x),
('roof', lambda x: 'roof' in x),
('gym', lambda x: 'gym' in x or 'fitness' in x),
('pool', lambda x: 'pool' in x),
('backyard', lambda x: 'backyard' in x),
('laundry', lambda x: 'laundry' in x),
('hardwood_floors', lambda x: 'hardwood_floors' in x),
('new_construction', lambda x: 'new_construction' in x),
('dryer', lambda x: 'dryer' in x),
('elevator', lambda x: 'elevator' in x),
('garage', lambda x: 'garage' in x),
('pre_war', lambda x: 'pre_war' in x or 'prewar' in x),
('post_war', lambda x: 'post_war' in x or 'postwar' in x),
('no_fee', lambda x: 'no_fee' in x),
('low_fee', lambda x: 'reduced_fee' in x or 'low_fee' in x),
('fire', lambda x: 'fireplace' in x),
('private', lambda x: 'private' in x),
('wheelchair', lambda x: 'wheelchair' in x),
('internet', lambda x: 'wifi' in x or 'wi_fi' in x or 'internet' in x),
('yoga', lambda x: 'yoga' in x),
('furnished', lambda x: 'furnished' in x),
('multi_level', lambda x: 'multi_level' in x),
('exclusive', lambda x: 'exclusive' in x),
('high_ceil', lambda x: 'high_ceil' in x),
('green', lambda x: 'green_b' in x),
('stainless', lambda x: 'stainless_' in x),
('simplex', lambda x: 'simplex' in x),
('public', lambda x: 'public' in x),
):
self.apply(df, k, condition)
df['bathrooms'] = df['bathrooms'].apply(lambda x: x if x < 5 else 5)
df['bedrooms'] = df['bedrooms'].apply(lambda x: x if x < 5 else 5)
df["num_photos"] = df["photos"].apply(len)
df["num_features"] = df["features"].apply(len)
created = pd.to_datetime(df.pop("created"))
df["listing_age"] = (pd.to_datetime('today') - created).apply(lambda x: x.days)
df["room_dif"] = df["bedrooms"] - df["bathrooms"]
df["room_sum"] = df["bedrooms"] + df["bathrooms"]
df["price_per_room"] = df["price"] / df["room_sum"].apply(lambda x: max(x, .5))
df["bedrooms_share"] = df["bedrooms"] / df["room_sum"].apply(lambda x: max(x, .5))
df['price'] = df['price'].apply(lambda x: np.log(x + EPSILON))
key_types = df.dtypes.to_dict()
for k in key_types:
if key_types[k].name not in ('int64', 'float64', 'int8'):
df.pop(k)
for k in ('latitude', 'longitude', 'listing_id'):
df.pop(k)
return df
def encode(x):
if x == 'low':
return 0
elif x == 'medium':
return 1
elif x == 'high':
return 2
def get_data():
with open('train.json', 'r') as raw_data:
data = json.load(raw_data)
df = pd.DataFrame(data)
target = df.pop('interest_level').apply(encode)
df = FeatureEngineer().fit_transform(df)
return df, target |