Kaggle competition: Rainforest Connection Species Audio Detection
<SOS> metric: WeightedLabelRankingAveragePrecision Kaggle data source: rainforest-connection-species-audio-detection<prepare_output>
import pandas as pd, numpy as np
import os
train_df['timestamp'] = train_df_csv['timestamp'].copy()
weather_df['timestamp'] = weather_df_csv['timestamp'].copy()
del weather_df_csv, train_df_csv
gc.collect()<feature_engineering>
paths = [
".. /input/rfcx-best-performing-public-kernels/kkiller_inference-tpu-rfcx-audio-detection-fast_0861.csv",
".. /input/rfcx-best-performing-public-kernels/submission_khoongweihao_0845.csv",
]
weights = np.array([0.6, 0.4])
 sum(weights )
def fill_weather(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 missing_hour_data = pd.DataFrame(np.setdiff1d(hours_list, site_hours), columns=['timestamp'])
 missing_hour_data['site_id'] = site_id
weather_df = pd.concat([weather_df, missing_hour_data])
 weather_df.reset_index(drop=True)
 weather_df['datetime'] = pd.to_datetime(weather_df['timestamp'])
 weather_df['day'] = weather_df['datetime'].dt.day
weather_df['week'] = weather_df['datetime'].dt.week
weather_df['month'] = weather_df['datetime'].dt.month
weather_df = weather_df.set_index(['site_id', 'day', 'month'])
 for col in ['air_temperature', 'cloud_coverage', 'dew_temperature', 'sea_level_pressure', 'wind_direction', 'wind_speed',\
'precip_depth_1_hr']:
mean_filter = weather_df.groupby(['site_id', 'day', 'month'])[col].mean()
mean_filter = pd.DataFrame(mean_filter.fillna(method='ffill'), columns=[col])
 weather_df.update(mean_filter, overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime', 'day', 'week', 'month'], axis=1)
 return weather_df<categorify>
cols = [f"s{i}" for i in range(24)]
def feature_engineering(df):
df.sort_values('timestamp')
 df.reset_index(drop=True)
 df["timestamp"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
 df["hour"] = df["timestamp"].dt.hour
df["weekday"] = df["timestamp"].dt.weekday
df['square_feet'] = np.log1p(df['square_feet'])
 drop = ["timestamp","sea_level_pressure", "wind_direction", "wind_speed","year_built","floor_count"]
df = df.drop(drop, axis=1)
 gc.collect()
le = LabelEncoder()
df["primary_use"] = le.fit_transform(df["primary_use"])
 return df<merge>
scores = []
for path in paths:
df = pd.read_csv(path ).sort_values("recording_id" ).reset_index(drop=True)
 score = np.empty(( len(df), 24))
o = df[cols].values.argsort(1)
 score[np.arange(len(df)) [:, None], o] = np.arange(24)[None]
scores.append(score)
 scores = np.stack(scores)
 scores.shape
weather_df = fill_weather(weather_df)
 train_df = reduce_mem_usage(train_df)
 building_df = reduce_mem_usage(building_df)
 weather_df = reduce_mem_usage(weather_df)
 train_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')
 train_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])
 del weather_df
gc.collect()
train_df = feature_engineering(train_df )<prepare_x_and_y>
sub_score = np.sum(scores*weights[:, None, None], 0)
 print(sub_score.shape)
 sub_score
train_X = train_df.drop(['meter_reading'], axis=1)
 train_y = np.log1p(train_df['meter_reading'])
 del train_df
gc.collect()<split>
sub = pd.DataFrame(sub_score, columns=cols)
 sub["recording_id"] = df["recording_id"]
sub = sub[["recording_id"] + cols]
print(sub.shape)
 sub.head()
<set_options><EOS> 
sub.to_csv("submission.csv", index=False )