Kaggle competition: Great Energy Predictor Shootout I
<SOS> metric: rmse Kaggle data source: great-energy-predictor-shootout-i<feature_engineering>
cf.set_config_file(offline=True)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
%%time
for i in range(0, 28):
day = fday + timedelta(days=i)
 print(i, day)
 tst = te[(te.date >= day - timedelta(days=max_lags)) &(te.date <= day)].copy()
create_fea(tst)
 tst = tst.loc[tst.date == day , train_cols]
te.loc[te.date == day, "sales"] = 1.02*m_lgb.predict(tst )<feature_engineering>
root = Path('.. /input/great-energy-predictor-shootout-i')
 df_submit = pd.read_csv(root/'sample_submission.csv')
 df_submit
%%time
te_sub = te.loc[te.date >= fday, ["id", "sales"]].copy()
te_sub.loc[te.date >= fday+ timedelta(days=h), "id"] = te_sub.loc[te.date >= fday+timedelta(days=h),
"id"].str.replace("validation$", "evaluation")
 te_sub["F"] = [f"F{rank}" for rank in te_sub.groupby("id")["id"].cumcount() +1]
te_sub = te_sub.set_index(["id", "F" ] ).unstack() ["sales"][[f"F{i}" for i in range(1,29)]].reset_index()
te_sub.fillna(0., inplace = True)
 te_sub.to_csv("submission.csv",index=False)
 te_sub.shape<install_modules>
df_test_A = pd.read_csv(root/'Atest.dat',delim_whitespace=True)
 df_test_A['Datetime'] = pd.to_datetime(( df_test_A['YEAR']+1900 ).astype('str')\
+'-'+(df_test_A['MONTH'] ).astype('str')\
+'-'+(df_test_A['DAY'] ).astype('str')\
+' '+(df_test_A['HOUR']/100 ).astype('int' ).astype('str')\
+':00')
 df_test_A.set_index('Datetime', inplace=True)
 df_test_A = df_test_A.drop(['MONTH','DAY','YEAR','HOUR'],axis=1)
 df_test_A['hourOfDay'] = df_test_A.index.hour
df_test_A['dayOfWeek'] = df_test_A.index.dayofweek
!pip install transformers<import_modules>
df_test_B = pd.read_csv(root/'Btest.dat',header=None,delim_whitespace=True)\ 
.rename(columns={0:'Date',1:'HorizRad',2:'SE_Rad',3:'S_Rad',4:'SW_Rad'})
 df_test_B.set_index('Date', inplace=True)
 df_test_B.reset_index(inplace=True)
 df_test_B['Day'] = np.floor(df_test_B['Date'])
 df_test_B['Time'] = df_test_B['Date'] - df_test_B['Day']
df_test_B.drop('Date', axis=1, inplace=True )
nltk.download('stopwords')
 stop_words = stopwords.words('english')
 nlp = spacy.load('en_core_web_sm' )<set_options>
df_train_A = pd.read_csv(root/'Atrain.dat',delim_whitespace=True)
 df_train_A['Datetime'] = pd.to_datetime(( df_train_A['YEAR']+1900 ).astype('str')\
+'-'+(df_train_A['MONTH'] ).astype('str')\
+'-'+(df_train_A['DAY'] ).astype('str')\
+' '+(df_train_A['HOUR']/100 ).astype('int' ).astype('str')\
+':00')
 df_train_A.set_index('Datetime', inplace=True)
 df_train_A = df_train_A.drop(['MONTH','DAY','YEAR','HOUR'],axis=1)
 df_train_A['hourOfDay'] = df_train_A.index.hour
df_train_A['dayOfWeek'] = df_train_A.index.dayofweek
df_train_A
print(torch.cuda.is_available() )<split>
df_train_B = pd.read_csv(root/'Btrain.dat',header=None,delim_whitespace=True)\ 
.rename(columns={0:'Date',1:'HorizRad',2:'SE_Rad',3:'S_Rad',4:'SW_Rad',5:'Beam_Rad'})
 df_train_B.set_index('Date', inplace=True)
 df_train_B.sort_index(inplace=True)
 df_train_B.reset_index(inplace=True)
 df_train_B['Day'] = np.floor(df_train_B['Date'])
 df_train_B['Time'] = df_train_B['Date'] - df_train_B['Day']
df_train_B.drop('Date', axis=1, inplace=True)
 df_train_B
def stratified_cv(classifier, X_train, y_train, **kwargs):

 skf = StratifiedKFold(**kwargs)
 total_accuracy = []
total_precision = []
total_recall = []
total_f1 = []
for train_index, val_index in skf.split(X_train, y_train):
current_X_train = X_train[train_index]
current_y_train = y_train.iloc[train_index]
current_X_val = X_train[val_index]
current_y_val = y_train.iloc[val_index]
clf = copy.deepcopy(classifier)
 clf.fit(current_X_train, current_y_train)
 current_predictions = clf.predict(current_X_val)
 total_accuracy.append(accuracy_score(current_y_val, current_predictions))
total_precision.append(precision_score(current_y_val, current_predictions))
total_recall.append(recall_score(current_y_val, current_predictions))
total_f1.append(f1_score(current_y_val, current_predictions))
ave_accuracy = mean(total_accuracy)
 ave_precision = mean(total_precision)
 ave_recall = mean(total_recall)
 ave_f1 = mean(total_f1)
 print("Average Accuracy: {:.4f}".format(ave_accuracy))
print("Average Precision: {:.4f}".format(ave_precision))
print("Average Recall: {:.4f}".format(ave_recall))
print("Average F1: {:.4f}".format(ave_f1))<categorify>
df_weather_merged = df_train_A[['TEMP','SOLAR']].append(df_test_A[['TEMP','SOLAR']])
 df_weather_merged = df_weather_merged.sort_index()
df_weather_merged
def clean_text(text):
text = re.sub(r"\x89Û_", "", text)
 text = re.sub(r"\x89ÛÒ", "", text)
 text = re.sub(r"\x89ÛÓ", "", text)
 text = re.sub(r"\x89ÛÏWhen", "When", text)
 text = re.sub(r"\x89ÛÏ", "", text)
 text = re.sub(r"China\x89Ûªs", "China's", text)
 text = re.sub(r"let\x89Ûªs", "let's", text)
 text = re.sub(r"\x89Û÷", "", text)
 text = re.sub(r"\x89Ûª", "", text)
 text = re.sub(r"\x89Û\x9d", "", text)
 text = re.sub(r"å_", "", text)
 text = re.sub(r"\x89Û¢", "", text)
 text = re.sub(r"\x89Û¢åÊ", "", text)
 text = re.sub(r"fromåÊwounds", "from wounds", text)
 text = re.sub(r"åÊ", "", text)
 text = re.sub(r"åÈ", "", text)
 text = re.sub(r"JapÌ_n", "Japan", text)
 text = re.sub(r"Ì©", "e", text)
 text = re.sub(r"å¨", "", text)
 text = re.sub(r"SuruÌ¤", "Suruc", text)
 text = re.sub(r"åÇ", "", text)
 text = re.sub(r"å£3million", "3 million", text)
 text = re.sub(r"åÀ", "", text)
 text = re.sub(r'https?://\S+|www\.\S+', '', text)
 text = re.sub(r'<.*?>', '', text)
 text = re.sub(r'[\d]+', ' ', text)
 return text
def preprocess(text, allowed_tags=["N", "J", "R", "V"]):
tag_map = defaultdict(lambda : wn.NOUN)
 tag_map['N'] = wn.NOUN
tag_map['J'] = wn.ADJ
tag_map['R'] = wn.ADV
tag_map['V'] = wn.VERB
text = text.lower()
tokens = [token for token in wordpunct_tokenize(text)]
lemmatizer = wn_lemmatizer()
lemmas = [lemmatizer.lemmatize(token, tag_map[tag[0]])for token, tag in pos_tag(tokens)if tag[0] in allowed_tags]
lemmas = [lemma for lemma in lemmas if lemma not in stop_words and lemma.isalpha() ]
return ' '.join(lemmas )<compute_test_metric>
alpha_data = {}
MAPE_data = {}
RSQUARED_data = {}
NMBE_data = {}
CVRSME_data = {}
data_test_A = df_test_A.copy()
data_test_B = df_test_B.copy()
def array_equal(array_1, array_2):

 if array_1.shape != array_2.shape:
return False
for i in range(len(array_1)) :
if(array_1[i] is array_2[i])or(array_1[i] == array_2[i]):
continue
return False
return True 
<load_from_csv>
for name_meter in ['WBE','WBCW','WBHW']:
data_train_A = df_train_A.copy()
data_train_A = data_train_A.merge(df_weather_merged, left_index=True, right_index=True)
 feat_train_A = data_train_A.drop(['WBE','WBCW','WBHW'],axis=1 ).copy()
label_train_A = data_train_A[name_meter].copy()
feat_test_A = df_test_A.copy()
feat_test_A = feat_test_A.merge(df_weather_merged, left_index=True, right_index=True)
 print('Power Meter: '+name_meter)
 LGB_model = LGBMRegressor()
data_train_A[name_meter + '_pred'] = cross_val_predict(LGB_model, feat_train_A, label_train_A, cv=5)
 LGB_model.fit(feat_train_A, label_train_A)
 data_test_A[name_meter + '_pred'] = LGB_model.predict(feat_test_A)
 errors = abs(data_train_A[name_meter + '_pred'] - label_train_A)
 errors_mean = round(np.mean(errors), 2)
 MAPE = 100 * np.mean(( errors / label_train_A))
NMBE = 100 *(sum(label_train_A - data_train_A[name_meter + '_pred'])/(pd.Series(label_train_A ).count() * np.mean(label_train_A)))
 CVRSME = 100 *(( sum(( label_train_A - data_train_A[name_meter + '_pred'])**2)/(pd.Series(label_train_A ).count() -1)) **(0.5)) / np.mean(label_train_A)
 RSQUARED = r2_score(label_train_A, data_train_A[name_meter + '_pred'])
 print("MAPE: "+str(round(MAPE,2)))
 print("NMBE: "+str(round(NMBE,2)))
 print("CVRSME: "+str(round(CVRSME,2)))
 print("R SQUARED: "+str(round(RSQUARED,2)))
 data_train_A[[name_meter,name_meter + '_pred']].iplot(kind='scatter', filename='cufflinks/cf-simple-line')
 print('-----------------------------------------------------------' )
data = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 data.head()<feature_engineering>
name_meter = 'Beam_Rad'
data_train_B = df_train_B.copy()
feat_train_B = data_train_B.drop('Beam_Rad',axis=1 ).copy()
label_train_B = data_train_B[name_meter].copy()
feat_test_B = df_test_B.copy()
print('Power Meter: '+name_meter)
 LGB_model = LGBMRegressor()
data_train_B[name_meter + '_pred'] = cross_val_predict(LGB_model, feat_train_B, label_train_B, cv=5)
 LGB_model.fit(feat_train_B, label_train_B)
 data_test_B[name_meter + '_pred'] = LGB_model.predict(feat_test_B)
 errors = abs(data_train_B[name_meter + '_pred'] - label_train_B)
 errors_mean = round(np.mean(errors), 2)
 MAPE = 100 * np.mean(( errors / label_train_B))
NMBE = 100 *(sum(label_train_B - data_train_B[name_meter + '_pred'])/(pd.Series(label_train_B ).count() * np.mean(label_train_B)))
 CVRSME = 100 *(( sum(( label_train_B - data_train_B[name_meter + '_pred'])**2)/(pd.Series(label_train_B ).count() -1)) **(0.5)) / np.mean(label_train_B)
 RSQUARED = r2_score(label_train_B, data_train_B[name_meter + '_pred'])
 print("MAPE: "+str(round(MAPE,2)))
 print("NMBE: "+str(round(NMBE,2)))
 print("CVRSME: "+str(round(CVRSME,2)))
 print("R SQUARED: "+str(round(RSQUARED,2)))
 data_train_B[[name_meter,name_meter + '_pred']].iplot(kind='scatter', filename='cufflinks/cf-simple-line')
 print('-----------------------------------------------------------' )
unique_targets = data.groupby('text' ).agg(unique_target=('target', pd.Series.nunique))
controversial_tweets = unique_targets[unique_targets['unique_target'] > 1].index
data = data[~data['text'].isin(controversial_tweets)]
data = data.drop_duplicates(subset='text', keep='first')
 data['text'] = data['text'].apply(clean_text)
 test['text'] = test['text'].apply(clean_text)
 data['value'] = 1
data['keyword'] = data['keyword'].fillna("nan")
 test['value'] = 1
test['keyword'] = test['keyword'].fillna("nan")
 data = pd.pivot_table(data, values="value", index=["id", "text", "target"], columns=["keyword"] ).fillna(0 ).reset_index()
test = pd.pivot_table(test, values="value", index=["id", "text"], columns=["keyword"] ).fillna(0 ).reset_index()<split>
df_submit.loc[df_submit['row_id'].str.endswith('WBE'), 'target'] = data_test_A[['WBE_pred']].melt() ['value'].values
df_submit.loc[df_submit['row_id'].str.endswith('WBCW'), 'target'] = data_test_A[['WBCW_pred']].melt() ['value'].values
df_submit.loc[df_submit['row_id'].str.endswith('WBHW'), 'target'] = data_test_A[['WBHW_pred']].melt() ['value'].values
df_submit.loc[df_submit['row_id'].str.endswith('true_beam_insolation'), 'target'] = data_test_B['Beam_Rad_pred'].values
df_submit
<categorify><EOS> 
df_submit.to_csv('submission.csv', index=False )