Kaggle competition: Kobe Bryant Shot Selection
train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv' )<load_pretrained>
def season_to_numeric(x):
return int(x.split('-')[0])- 1996
data["season"] = data["season"].apply(season_to_numeric )
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train.PetID.values:
try:
with open('.. /input/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1 )<feature_engineering>
def home_or_away(x):
if "@" in x:
return -1
elif "vs." in x:
return 1
else:
return 0
data['matchup'] = data['matchup'].apply(home_or_away )
train['doc_sent_mag'] = doc_sent_mag
train['doc_sent_score'] = doc_sent_score<load_from_disk>
def time_from_start(x):
if x["period"] <= 4:
period_minutes = x["period"] * 12
else:
period_minutes = 48 +(x["period"] - 4)* 5
return(period_minutes - x["minutes_remaining"])* 60 - x["seconds_remaining"]
data['time_from_start'] = data.apply(time_from_start, axis = 1 )
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test.PetID.values:
try:
with open('.. /input/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1 )<feature_engineering>
def categorical_to_freq(feature_name):
my_counts = data[feature_name].value_counts()
return data[feature_name].apply(lambda x: my_counts[x])
 action_type = categorical_to_freq("action_type")
 combined_shot_type = categorical_to_freq("combined_shot_type")
 shot_type = categorical_to_freq("shot_type")
 shot_zone_area = categorical_to_freq("shot_zone_area")
 shot_zone_basic = categorical_to_freq("shot_zone_basic")
 shot_zone_range = categorical_to_freq("shot_zone_range")
test['doc_sent_mag'] = doc_sent_mag
test['doc_sent_score'] = doc_sent_score<categorify>

 mydata = data[["matchup","season", "time_from_start", "game_event_id", "game_id", "lat", "loc_x", "loc_y", "lon", "minutes_remaining", "period", "playoffs", "seconds_remaining", "shot_distance", "shot_made_flag", "shot_id"]]
mydata = pd.concat([mydata, action_type, combined_shot_type, shot_type, shot_zone_area, shot_zone_basic, shot_zone_range], axis = 1)
 train = mydata[mydata["shot_made_flag"].notnull() ]
test = mydata[mydata["shot_made_flag"].isnull() ]
target = train["shot_made_flag"]
train = train.drop(["shot_made_flag", "shot_id"], 1)
 test_shot_id = test["shot_id"]
test = test.drop(["shot_made_flag", "shot_id"], 1)
lbl_enc = LabelEncoder()
lbl_enc.fit(train.RescuerID.values.tolist() + test.RescuerID.values.tolist())
 train.RescuerID = lbl_enc.transform(train.RescuerID.values)
 test.RescuerID = lbl_enc.transform(test.RescuerID.values )<normalization>
gbrt = GradientBoostingRegressor(n_estimators=100, learning_rate=0.05, max_depth=5, random_state=0, loss='ls' ).fit(train, target)
 gbrt_res = gbrt.predict(test )
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=3, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc)+ list(test_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=180)
 svd.fit(X)
 X = svd.transform(X)
 X_test = svd.transform(X_test )<prepare_x_and_y>
output = pd.concat([test_shot_id.reset_index(drop = True), pd.Series(data = gbrt_res, name = 'shot_made_flag')], axis = 1)
 output.to_csv("my_third_submission.csv", index = False)
 print(check_output(["cp", ".. /input/sample_submission.csv", "sample_submission.csv"] ).decode("utf8"))
<count_values><EOS> 
print(check_output(["head", "my_third_submission.csv"] ).decode("utf8"))