I am working on the file "04_time_series_prediction/11_price_history_dummy_seq2seq_EOS_zero.ipynb".
The first task is:
Step 3 training the network
RECALL: baseline is around 4 for huber loss for current problem, anything above 4 should be considered as major errors
Basic RNN cell without predictions - with EOS = 0
Can you write Python code for it?

rnn_cell = PriceHistorySeq2Seq.RNN_CELLS.BASIC_RNN
num_epochs = 50
num_epochs, num_units, batch_size

def experiment():
    return model.run(
        npz_path=npz_path,
        epochs=num_epochs,
        batch_size=batch_size,
        num_units=num_units,
        input_len = input_len,
        target_len = target_len,
        rnn_cell=rnn_cell,
    )

dyn_stats = experiment()
#dyn_stats, preds_dict = get_or_run_nn(experiment, filename='003_rnn_slidewin_60to1')

dyn_stats.plotStats()
plt.show()


I am working on the file "04_time_series_prediction/11_price_history_dummy_seq2seq_EOS_zero.ipynb".
The first task is:
https://www.youtube.com/watch?v=ElmBrKyMXxs
https://github.com/hans/ipython-notebooks/blob/master/tf/TF%20tutorial.ipynb
https://github.com/ematvey/tensorflow-seq2seq-tutorials
Can you write Python code for it?

from __future__ import division
import tensorflow as tf
from os import path
import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import StratifiedShuffleSplit
from time import time
from matplotlib import pyplot as plt
import seaborn as sns
from mylibs.jupyter_notebook_helper import show_graph
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
import shutil
from tensorflow.contrib.learn.python.learn import learn_runner
from mylibs.tf_helper import getDefaultGPUconfig
from sklearn.metrics import r2_score
from mylibs.py_helper import factors
from fastdtw import fastdtw
from scipy.spatial.distance import euclidean
from statsmodels.tsa.stattools import coint
from common import get_or_run_nn
from data_providers.price_history_dummy_seq2seq_data_provider \
    import PriceHistoryDummySeq2SeqDataProvider
from models.price_history_dummy_seq2seq import PriceHistoryDummySeq2Seq

dtype = tf.float32
seed = 16011984
random_state = np.random.RandomState(seed=seed)
config = getDefaultGPUconfig()
%matplotlib inline


I am working on the file "04_time_series_prediction/12_price_history_dummy_seq2seq_with_and_without_EOS.ipynb".
The first task is:
Conclusion
Now there is no shifting but this only works for the dummy case where the decoder inputs are provider from the targets (which is incorrect because the targets should be unknown in general)
GRU cell - without EOS
Can you write Python code for it?

rnn_cell = PriceHistoryDummySeq2Seq.RNN_CELLS.GRU
num_epochs = 10
num_epochs, num_units, batch_size

def experiment():
    return model.run(
        npz_path=npz_path,
        epochs=num_epochs,
        batch_size=batch_size,
        num_units=num_units,
        input_len = input_len,
        target_len = target_len,
        rnn_cell=rnn_cell,
    )

#dyn_stats = experiment()
dyn_stats, preds_dict = get_or_run_nn(experiment, filename='005_gru_dummy_seq2seq_noEOS_60to30')

dyn_stats.plotStats()
plt.show()

r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]

ind = np.argmin(r2_scores)
ind

reals = dp.targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


I am working on the file "04_time_series_prediction/12_price_history_dummy_seq2seq_with_and_without_EOS.ipynb".
The first task is:
Conclusion
there is a good fit
GRU cell - with EOS = 1000
Can you write Python code for it?

rnn_cell = PriceHistoryDummySeq2Seq.RNN_CELLS.GRU
num_epochs = 30
num_epochs, num_units, batch_size

def experiment():
    return model.run(
        npz_path=npz_path,
        epochs=num_epochs,
        batch_size=batch_size,
        num_units=num_units,
        input_len = input_len,
        target_len = target_len,
        rnn_cell=rnn_cell,
    )

#dyn_stats = experiment()
dyn_stats, preds_dict = get_or_run_nn(experiment, filename='005_gru_dummy_seq2seq_EOS1000_60to30')

dyn_stats.plotStats()
plt.show()

r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]

ind = np.argmin(r2_scores)
ind

reals = dp.targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


I am working on the file "04_time_series_prediction/12_price_history_dummy_seq2seq_with_and_without_EOS.ipynb".
The first task is:
Step 2 - Build model
Can you write Python code for it?

model = PriceHistoryDummySeq2Seq(rng=get_init_rng(), dtype=dtype, config=config, with_EOS=False)

graph = model.getGraph(batch_size=batch_size,
                       num_units=num_units,
                       input_len=input_len,
                       target_len=target_len)

#show_graph(graph)


I am working on the file "04_time_series_prediction/12_price_history_dummy_seq2seq_with_and_without_EOS.ipynb".
The first task is:
Step 3 training the network
RECALL: baseline is around 4 for huber loss for current problem, anything above 4 should be considered as major errors
Basic RNN cell - with EOS 1000
Can you write Python code for it?

rnn_cell = PriceHistoryDummySeq2Seq.RNN_CELLS.BASIC_RNN
num_epochs = 30
num_epochs, num_units, batch_size

def experiment():
    return model.run(
        npz_path=npz_path,
        epochs=num_epochs,
        batch_size=batch_size,
        num_units=num_units,
        input_len = input_len,
        target_len = target_len,
        rnn_cell=rnn_cell,
    )

dyn_stats, preds_dict = get_or_run_nn(experiment, filename='005_basic_rnn_dummy_seq2seq_EOS1000_60to30')

dyn_stats.plotStats()
plt.show()

r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]

ind = np.argmin(r2_scores)
ind

reals = dp.targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


I am working on the file "04_time_series_prediction/13_price_history_seq2seq-raw.ipynb".
The first task is:
Conclusion
???
Basic RNN cell (EOS 1000)
Can you write Python code for it?

rnn_cell = PriceHistorySeq2Seq.RNN_CELLS.BASIC_RNN
num_epochs = 10
num_epochs, num_units, batch_size

def experiment():
    return model.run(
        npz_path=npz_path,
        epochs=num_epochs,
        batch_size=batch_size,
        num_units=num_units,
        input_len = input_len,
        target_len = target_len,
        rnn_cell=rnn_cell,
    )

dyn_stats, preds_dict = get_or_run_nn(experiment, filename='006_basic_rnn_seq2seq_EOS1000_60to30')


I am working on the file "04_time_series_prediction/13_price_history_seq2seq-raw.ipynb".
The first task is:
Conclusion
It is oscillating which is an indication that setting EOS to zero where the real values is not helping the model
Perhaps another idea is to have as first input of the decoder RNN the latest value of the encoding stream
GRU cell - with EOS = 0
Can you write Python code for it?

rnn_cell = PriceHistorySeq2Seq.RNN_CELLS.GRU
num_epochs = 10
num_epochs, num_units, batch_size

def experiment():
    return model.run(
        npz_path=npz_path,
        epochs=num_epochs,
        batch_size=batch_size,
        num_units=num_units,
        input_len = input_len,
        target_len = target_len,
        rnn_cell=rnn_cell,
        eos_token = 0,
    )

#dyn_stats = experiment()
dyn_stats, preds_dict = get_or_run_nn(experiment, filename='006_gru_seq2seq_zeroEOS_60to30')

dyn_stats.plotStats()
plt.show()

r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]

ind = np.argmin(r2_scores)
ind

reals = dp.targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


I am working on the file "04_time_series_prediction/13_price_history_seq2seq-raw.ipynb".
The first task is:
GRU cell - with EOS = 1000 - 50 epochs
Can you write Python code for it?

rnn_cell = PriceHistorySeq2Seq.RNN_CELLS.GRU
num_epochs = 50
num_epochs, num_units, batch_size

def experiment():
    return model.run(
        npz_path=npz_path,
        epochs=num_epochs,
        batch_size=batch_size,
        num_units=num_units,
        input_len = input_len,
        target_len = target_len,
        rnn_cell=rnn_cell,
    )

#dyn_stats = experiment()
dyn_stats, preds_dict = get_or_run_nn(experiment, filename='006_gru_seq2seq_EOS1000_50epochs_60to30')

dyn_stats.plotStats()
plt.show()

r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]

ind = np.argmin(r2_scores)
ind

reals = dp.targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


I am working on the file "04_time_series_prediction/13_price_history_seq2seq-raw.ipynb".
The first task is:
Step 0 - hyperparams
vocab_size is all the potential words you could have (classification for translation case)
and max sequence length are the SAME thing
decoder RNN hidden units are usually same size as encoder RNN hidden units in translation but for our case it does not seem really to be a relationship there but we can experiment and find out later, not a priority thing right now
Can you write Python code for it?

num_epochs = 10

num_features = 1
num_units = 400 #state size

input_len = 60
target_len = 30

batch_size = 47

#trunc_backprop_len = ??


I am working on the file "04_time_series_prediction/13_price_history_seq2seq-raw.ipynb".
The first task is:
Step 1 - collect data (and/or generate them)
Can you write Python code for it?

npz_path = '../price_history_03_dp_60to30_from_fixed_len.npz'

dp = PriceHistorySeq2SeqDataProvider(npz_path=npz_path, batch_size=batch_size)
dp.inputs.shape, dp.targets.shape

aa, bb = dp.next()
aa.shape, bb.shape


I am working on the file "04_time_series_prediction/13_price_history_seq2seq-raw.ipynb".
The first task is:
Step 2 - Build model
```
PriceHistoryDummySeq2Seq(rng=random_state, dtype=dtype, config=config).getGraph(batch_size=batch_size,
                       num_units=num_units,
                       input_len=input_len,
                       target_len=target_len)
learning rate: 0.001000
60
Tensor("inputs/unstack:0", shape=(47, 1), dtype=float32)
Tensor("encoder_rnn_layer/rnn/basic_rnn_cell_59/Tanh:0", shape=(47, 400), dtype=float32)
decoder inputs series
<type 'list'>
30
Tensor("decoder_rnn_layer/unstack:0", shape=(47, 1), dtype=float32)
30
Tensor("decoder_rnn_layer/rnn/basic_rnn_cell/Tanh:0", shape=(47, 400), dtype=float32)
30
Tensor("readout_layer/add:0", shape=(47, 1), dtype=float32)
Tensor("predictions/Reshape:0", shape=(47, 30), dtype=float32)
Tensor("error/Select:0", shape=(47, 30), dtype=float32)
Tensor("error/Mean:0", shape=(), dtype=float32)
```
Can you write Python code for it?

model = PriceHistorySeq2Seq(rng=random_state, dtype=dtype, config=config)

# graph = model.getGraph(batch_size=batch_size,
#                        num_units=num_units,
#                        input_len=input_len,
#                        target_len=target_len)


I am working on the file "04_time_series_prediction/14_price_history_seq2seq-native.ipynb".
The first task is:
Conclusion
The initial price difference of the predictions is still not as good as we would expect, perhaps using an EOS as they do in machine translation models is not the best architecture for our case
GRU cell - with EOS = 1000
Can you write Python code for it?

rnn_cell = PriceHistorySeq2SeqNative.RNN_CELLS.GRU
num_epochs = 30
eos_token = float(1e3)
num_epochs, num_units, batch_size

def experiment():
    return model.run(
        npz_path=npz_path,
        epochs=num_epochs,
        batch_size=batch_size,
        num_units=num_units,
        input_len = input_len,
        target_len = target_len,
        rnn_cell=rnn_cell,
        eos_token=eos_token,
    )

experiment()

dyn_stats, preds_dict = get_or_run_nn(experiment, filename='007_gru_seq2seq_native_EOS1000_60to30_30epochs')

dyn_stats.plotStats()
plt.show()


I am working on the file "04_time_series_prediction/14_price_history_seq2seq-native.ipynb".
The first task is:
Step 1 - collect data (and/or generate them)
Can you write Python code for it?

npz_path = '../price_history_03_dp_60to30_from_fixed_len.npz'
#npz_path = '../data/price_history/price_history_03_dp_60to30_6400_train.npz'

dp = PriceHistorySeq2SeqDataProvider(npz_path=npz_path, batch_size=batch_size)
dp.inputs.shape, dp.targets.shape

aa, bb = dp.next()
aa.shape, bb.shape


I am working on the file "04_time_series_prediction/14_price_history_seq2seq-native.ipynb".
The first task is:
Step 2 - Build model
Can you write Python code for it?

model = PriceHistorySeq2SeqNative(rng=random_state, dtype=dtype, config=config)

graph = model.getGraph(batch_size=batch_size,
                       num_units=num_units,
                       input_len=input_len,
                       target_len=target_len)

#show_graph(graph)


I am working on the file "04_time_series_prediction/14_price_history_seq2seq-native.ipynb".
The first task is:
https://www.youtube.com/watch?v=ElmBrKyMXxs
https://github.com/hans/ipython-notebooks/blob/master/tf/TF%20tutorial.ipynb
https://github.com/ematvey/tensorflow-seq2seq-tutorials
Can you write Python code for it?

from __future__ import division
import tensorflow as tf
from os import path
import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import StratifiedShuffleSplit
from time import time
from matplotlib import pyplot as plt
import seaborn as sns
from mylibs.jupyter_notebook_helper import show_graph
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
import shutil
from tensorflow.contrib.learn.python.learn import learn_runner
from mylibs.tf_helper import getDefaultGPUconfig
from sklearn.metrics import r2_score
from mylibs.py_helper import factors
from fastdtw import fastdtw
from scipy.spatial.distance import euclidean
from statsmodels.tsa.stattools import coint
from common import get_or_run_nn
from data_providers.price_history_seq2seq_data_provider import PriceHistorySeq2SeqDataProvider
from models.price_history_seq2seq_native import PriceHistorySeq2SeqNative

dtype = tf.float32
seed = 16011984
random_state = np.random.RandomState(seed=seed)
config = getDefaultGPUconfig()
%matplotlib inline


I am working on the file "04_time_series_prediction/15_price_history_seq2seq-native-with-last-input-as-decoder-input.ipynb".
The first task is:
Conclusion
There is no way this graph makes much sense but let's give it a try to see how bad really is
Step 3 training the network
RECALL: baseline is around 4 for huber loss for current problem, anything above 4 should be considered as major errors
Basic RNN cell (without EOS)
Can you write Python code for it?

model = PriceHistorySeq2SeqNative(rng=random_state, dtype=dtype, config=config, with_EOS=False)

rnn_cell = PriceHistorySeq2SeqNative.RNN_CELLS.BASIC_RNN
num_epochs = 20
num_epochs, num_units, batch_size

def experiment():
    return model.run(
        npz_path=npz_path,
        epochs=num_epochs,
        batch_size=batch_size,
        num_units=num_units,
        input_len = input_len,
        target_len = target_len,
        rnn_cell=rnn_cell,
    )

dyn_stats, preds_dict = get_or_run_nn(experiment, filename='008_rnn_seq2seq_native_noEOS_60to30_20epochs')

dyn_stats.plotStats()
plt.show()

r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]

ind = np.argmin(r2_scores)
ind

reals = dp.targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


I am working on the file "04_time_series_prediction/15_price_history_seq2seq-native-with-last-input-as-decoder-input.ipynb".
The first task is:
Step 1 - collect data (and/or generate them)
Can you write Python code for it?

npz_path = '../price_history_03_dp_60to30_from_fixed_len.npz'

dp = PriceHistorySeq2SeqDataProvider(npz_path=npz_path, batch_size=batch_size, with_EOS=False)
dp.inputs.shape, dp.targets.shape

aa, bb = dp.next()
aa.shape, bb.shape


I am working on the file "04_time_series_prediction/15_price_history_seq2seq-native-with-last-input-as-decoder-input.ipynb".
The first task is:
https://www.youtube.com/watch?v=ElmBrKyMXxs
https://github.com/hans/ipython-notebooks/blob/master/tf/TF%20tutorial.ipynb
https://github.com/ematvey/tensorflow-seq2seq-tutorials
Can you write Python code for it?


I am working on the file "04_time_series_prediction/17_price_history_seq2seq-overfitting.ipynb".
The first task is:
Recall that without batch normalization within 10 epochs with num units 400 and batch_size 64 we reached at 4.940
and with having the decoder inputs NOT filled from the outputs
Can you write Python code for it?

%%time
dyn_stats, preds_dict = get_or_run_nn(experiment,
                                      filename='017_seq2seq_60to30_epochs{}_learning_rate_{:.4f}'.format(
                                          epochs, best_learning_rate
                                      ))

dyn_stats.plotStats()
plt.show()

r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]

ind = np.argmin(r2_scores)
ind

reals = dp.targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


I am working on the file "04_time_series_prediction/17_price_history_seq2seq-overfitting.ipynb".
The first task is:
Step 0 - hyperparams
vocab_size is all the potential words you could have (classification for translation case)
and max sequence length are the SAME thing
decoder RNN hidden units are usually same size as encoder RNN hidden units in translation but for our case it does not seem really to be a relationship there but we can experiment and find out later, not a priority thing right now
Can you write Python code for it?

epochs = 15

num_features = 1
num_units = 400 #state size

input_len = 60
target_len = 30

batch_size = 50 #47
#trunc_backprop_len = ??
rnn_cell = PriceHistorySeq2SeqRawDummy.RNN_CELLS.GRU

with_EOS = False

total_train_size = 57994
train_size = 6400
test_size = 1282


I am working on the file "04_time_series_prediction/17_price_history_seq2seq-overfitting.ipynb".
The first task is:
Step 1 - collect data
Can you write Python code for it?

dp = PriceHistorySeq2SeqDataProvider(npz_path=npz_train, batch_size=batch_size, with_EOS=with_EOS)
dp.inputs.shape, dp.targets.shape

aa, bb = dp.next()
aa.shape, bb.shape


I am working on the file "04_time_series_prediction/17_price_history_seq2seq-overfitting.ipynb".
The first task is:
Step 2 - Build model
Can you write Python code for it?

model = PriceHistorySeq2SeqRawDummy(rng=random_state, dtype=dtype, config=config, with_EOS=with_EOS)

graph = model.getGraph(batch_size=batch_size,
                       num_units=num_units,
                       input_len=input_len,
                       target_len=target_len,
                       rnn_cell=rnn_cell)

#show_graph(graph)


I am working on the file "04_time_series_prediction/17_price_history_seq2seq-overfitting.ipynb".
The first task is:
Step 3 training the network
RECALL: baseline is around 4 for huber loss for current problem, anything above 4 should be considered as major errors
Can you write Python code for it?

#rnn_cell = PriceHistorySeq2SeqCV.RNN_CELLS.GRU
#cross_val_n_splits = 5
epochs, num_units, batch_size

#set(factors(train_size)).intersection(factors(train_size/5))

best_learning_rate = 1e-3 #0.0026945952539362472

def experiment():
    return model.run(npz_path=npz_train,
             epochs=10,
             batch_size = 50,
             num_units = 400,
             input_len=input_len,
             target_len=target_len,
             learning_rate = best_learning_rate,
             preds_gather_enabled=True,
             #eos_token = float(1e3),
             rnn_cell=rnn_cell)

dyn_stats, preds_dict = experiment()


I am working on the file "04_time_series_prediction/18_price_history_seq2seq-batch_norm.ipynb".
The first task is:
Once generate data
Can you write Python code for it?

data_path = '../data/price_history'

#npz_full_train = data_path + '/price_history_03_dp_60to30_train.npz'
#npz_full_train = data_path + '/price_history_60to30_targets_normed_train.npz'

#npz_train = data_path + '/price_history_03_dp_60to30_57980_train.npz'
#npz_train = data_path + '/price_history_03_dp_60to30_6400_train.npz'
npz_train = data_path + '/price_history_60to30_6400_targets_normed_train.npz'

#npz_test = data_path + '/price_history_03_dp_60to30_test.npz'
npz_test = data_path + '/price_history_60to30_targets_normed_test.npz'


I am working on the file "04_time_series_prediction/18_price_history_seq2seq-batch_norm.ipynb".
The first task is:
Step 1 - collect data
Can you write Python code for it?


I am working on the file "04_time_series_prediction/18_price_history_seq2seq-batch_norm.ipynb".
The first task is:
Step 3 training the network
RECALL: baseline is around 4 for huber loss for current problem, anything above 4 should be considered as major errors
Can you write Python code for it?

#rnn_cell = PriceHistorySeq2SeqCV.RNN_CELLS.GRU
#cross_val_n_splits = 5
epochs, num_units, batch_size

#set(factors(train_size)).intersection(factors(train_size/5))

best_learning_rate = 1e-3 #0.0026945952539362472

def experiment():
    return model.run(npz_path=npz_train,
             epochs=10,
             batch_size = 50,
             num_units = 400,
             input_len=input_len,
             target_len=target_len,
             learning_rate = best_learning_rate,
             preds_gather_enabled=True,
             #eos_token = float(1e3),
            )


I am working on the file "04_time_series_prediction/18_price_history_seq2seq-batch_norm.ipynb".
The first task is:
https://www.youtube.com/watch?v=ElmBrKyMXxs
https://github.com/hans/ipython-notebooks/blob/master/tf/TF%20tutorial.ipynb
https://github.com/ematvey/tensorflow-seq2seq-tutorials
Can you write Python code for it?

from __future__ import division
import tensorflow as tf
from os import path, remove
import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import StratifiedShuffleSplit
from time import time
from matplotlib import pyplot as plt
import seaborn as sns
from mylibs.jupyter_notebook_helper import show_graph, renderStatsList, renderStatsCollection, \
    renderStatsListWithLabels, renderStatsCollectionOfCrossValids
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
import shutil
from tensorflow.contrib.learn.python.learn import learn_runner
from mylibs.tf_helper import getDefaultGPUconfig
from sklearn.metrics import r2_score
from mylibs.py_helper import factors
from fastdtw import fastdtw
from collections import OrderedDict
from scipy.spatial.distance import euclidean
from statsmodels.tsa.stattools import coint
from common import get_or_run_nn
from data_providers.price_history_seq2seq_data_provider import PriceHistorySeq2SeqDataProvider
from data_providers.price_history_dataset_generator import PriceHistoryDatasetGenerator
from skopt.space.space import Integer, Real
from skopt import gp_minimize
from skopt.plots import plot_convergence
import pickle
import inspect
import dill
import sys
from models.price_history_18_seq2seq_raw_batch_norm import PriceHistorySeq2SeqRawBatchNorm

dtype = tf.float32
seed = 16011984
random_state = np.random.RandomState(seed=seed)
config = getDefaultGPUconfig()
n_jobs = 1
%matplotlib inline


I am working on the file "04_time_series_prediction/19_price_history_seq2seq-dropout.ipynb".
The first task is:
Once generate data
Can you write Python code for it?


I am working on the file "04_time_series_prediction/19_price_history_seq2seq-dropout.ipynb".
The first task is:
Recall that without batch normalization within 10 epochs with num units 400 and batch_size 64 we reached at 4.940
and with having the decoder inputs NOT filled from the outputs
Can you write Python code for it?

%%time
dyn_stats, preds_dict = get_or_run_nn(
  experiment,
  filename='019_seq2seq_60to30_epochs{}_learning_rate_{:.4f}_prob_input{}'.format(
      epochs, best_learning_rate, keep_prob_input
  ))

dyn_stats.plotStats()
plt.show()

r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]

ind = np.argmin(r2_scores)
ind

reals = dp.targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


I am working on the file "04_time_series_prediction/19_price_history_seq2seq-dropout.ipynb".
The first task is:
Step 0 - hyperparams
vocab_size is all the potential words you could have (classification for translation case)
and max sequence length are the SAME thing
decoder RNN hidden units are usually same size as encoder RNN hidden units in translation but for our case it does not seem really to be a relationship there but we can experiment and find out later, not a priority thing right now
Can you write Python code for it?

epochs = 10

num_features = 1
num_units = 400 #state size

input_len = 60
target_len = 30

batch_size = 50 #47
#trunc_backprop_len = ??
with_EOS = False

total_train_size = 57994
train_size = 6400
test_size = 1282


I am working on the file "04_time_series_prediction/19_price_history_seq2seq-dropout.ipynb".
The first task is:
Step 3 training the network
RECALL: baseline is around 4 for huber loss for current problem, anything above 4 should be considered as major errors
Can you write Python code for it?

#rnn_cell = PriceHistorySeq2SeqCV.RNN_CELLS.GRU
#cross_val_n_splits = 5
epochs, num_units, batch_size

#set(factors(train_size)).intersection(factors(train_size/5))

best_learning_rate = 1e-3 #0.0026945952539362472
keep_prob_input = 0.7

def experiment():
    return model.run(npz_path=npz_train,
             epochs=epochs,
             batch_size = batch_size,
             num_units = num_units,
             input_len=input_len,
             target_len=target_len,
             learning_rate = best_learning_rate,
             preds_gather_enabled=True,
             keep_prob_input = keep_prob_input,
            )


I am working on the file "04_time_series_prediction/19_price_history_seq2seq-dropout.ipynb".
The first task is:
https://www.youtube.com/watch?v=ElmBrKyMXxs
https://github.com/hans/ipython-notebooks/blob/master/tf/TF%20tutorial.ipynb
https://github.com/ematvey/tensorflow-seq2seq-tutorials
Can you write Python code for it?

from __future__ import division
import tensorflow as tf
from os import path, remove
import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import StratifiedShuffleSplit
from time import time
from matplotlib import pyplot as plt
import seaborn as sns
from mylibs.jupyter_notebook_helper import show_graph, renderStatsList, renderStatsCollection, \
    renderStatsListWithLabels, renderStatsCollectionOfCrossValids
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
import shutil
from tensorflow.contrib.learn.python.learn import learn_runner
from mylibs.tf_helper import getDefaultGPUconfig
from sklearn.metrics import r2_score
from mylibs.py_helper import factors
from fastdtw import fastdtw
from collections import OrderedDict
from scipy.spatial.distance import euclidean
from statsmodels.tsa.stattools import coint
from common import get_or_run_nn
from data_providers.price_history_seq2seq_data_provider import PriceHistorySeq2SeqDataProvider
from data_providers.price_history_dataset_generator import PriceHistoryDatasetGenerator
from skopt.space.space import Integer, Real
from skopt import gp_minimize
from skopt.plots import plot_convergence
import pickle
import inspect
import dill
import sys
from models.price_history_19_seq2seq_raw_dropout import PriceHistorySeq2SeqRawDropout

dtype = tf.float32
seed = 16011984
random_state = np.random.RandomState(seed=seed)
config = getDefaultGPUconfig()
n_jobs = 1
%matplotlib inline


I am working on the file "04_time_series_prediction/20_price_history_seq2seq-L2reg.ipynb".
The first task is:
Once generate data
Can you write Python code for it?


I am working on the file "04_time_series_prediction/20_price_history_seq2seq-L2reg.ipynb".
The first task is:
Step 0 - hyperparams
vocab_size is all the potential words you could have (classification for translation case)
and max sequence length are the SAME thing
decoder RNN hidden units are usually same size as encoder RNN hidden units in translation but for our case it does not seem really to be a relationship there but we can experiment and find out later, not a priority thing right now
Can you write Python code for it?

epochs = 10

num_features = 1
num_units = 400 #state size

input_len = 60
target_len = 30
lamda2 = 1e-2

batch_size = 50 #47
#trunc_backprop_len = ??
with_EOS = False

total_train_size = 57994
train_size = 6400
test_size = 1282


I am working on the file "04_time_series_prediction/20_price_history_seq2seq-L2reg.ipynb".
The first task is:
Step 2 - Build model
Can you write Python code for it?

model = PriceHistorySeq2SeqRawL2reg(rng=random_state, dtype=dtype, config=config, with_EOS=with_EOS)

graph = model.getGraph(batch_size=batch_size,
                       num_units=num_units,
                       input_len=input_len,
                       target_len=target_len,
                       lamda2=lamda2)

#show_graph(graph)


I am working on the file "04_time_series_prediction/20_price_history_seq2seq-L2reg.ipynb".
The first task is:
Step 3 training the network
RECALL: baseline is around 4 for huber loss for current problem, anything above 4 should be considered as major errors
Can you write Python code for it?

#rnn_cell = PriceHistorySeq2SeqCV.RNN_CELLS.GRU
#cross_val_n_splits = 5
epochs, num_units, batch_size

#set(factors(train_size)).intersection(factors(train_size/5))

best_learning_rate = 1e-3 #0.0026945952539362472
keep_prob_input = 0.7

def experiment():
    return model.run(npz_path=npz_train,
             epochs=epochs,
             batch_size = batch_size,
             num_units = num_units,
             input_len=input_len,
             target_len=target_len,
             learning_rate = best_learning_rate,
             preds_gather_enabled=True,
             keep_prob_input = keep_prob_input,
             lamda2=lamda2,
            )


I am working on the file "04_time_series_prediction/22_price_history_seq2seq-removed_scale.ipynb".
The first task is:
Conclusion
???
Global Scale Normalization plus normalization per instance
Once generate data
Can you write Python code for it?

data_path = '../data/price_history'

#npz_full_train = data_path + '/price_history_03_dp_60to30_train.npz'
#npz_full_train = data_path + '/price_history_60to30_targets_normed_train.npz'
npz_full_train = data_path + '/price_history_03_dp_60to30_global_local_remove_scale_train.npz'

#npz_train = data_path + '/price_history_03_dp_60to30_57980_train.npz'
#npz_train = data_path + '/price_history_03_dp_60to30_6400_train.npz'
#npz_train = data_path + '/price_history_60to30_6400_targets_normed_train.npz'
npz_train = data_path + '/price_history_03_dp_60to30_6400_global_local_remove_scale_train.npz'

#npz_test = data_path + '/price_history_03_dp_60to30_test.npz'
#npz_test = data_path + '/price_history_60to30_targets_normed_test.npz'
npz_test = data_path + '/price_history_03_dp_60to30_global_local_remove_scale_test.npz'

#%%time
# csv_in = '../price_history_03_seq_start_suddens_trimmed.csv'

# train_sku_ids, train_XX, train_YY, train_sequence_lens, train_seq_mask, test_pack = \
#     PriceHistoryDatasetGenerator(random_state=random_state).createAndSaveDataset(
#             csv_in=csv_in,
#             input_seq_len=input_len,
#             target_seq_len=target_len,
#             allowSmallerSequencesThanWindow=False,
#             #min_date = '2016-11-01',
#             split_fraction = 0.40,
#             #keep_training_fraction = 0.22, #57994 * 0.22 = 12758.68
#             normalize_targets = False,
#             do_global_norm_scale = True,
#             norm_instance_scale = True,
#             #disable saving for now since we have already created them
#             save_files_dic = {"train": npz_full_train, "test": npz_test,},
#     )

# print train_sku_ids.shape, train_XX.shape, train_YY.shape, train_sequence_lens.shape, train_seq_mask.shape
# aa,bb,cc,dd,ee = test_pack.get_data()
# aa.shape,bb.shape,cc.shape,dd.shape,ee.shape

# PriceHistoryDatasetGenerator.create_subsampled(inpath=npz_full_train,
#                                                target_size = train_size,
#                                      ...


I am working on the file "04_time_series_prediction/22_price_history_seq2seq-removed_scale.ipynb".
The first task is:
Global Scale Normalization
Once generate data
Can you write Python code for it?

data_path = '../data/price_history'

#npz_full_train = data_path + '/price_history_03_dp_60to30_train.npz'
#npz_full_train = data_path + '/price_history_60to30_targets_normed_train.npz'
npz_full_train = data_path + '/price_history_03_dp_60to30_global_remove_scale_train.npz'

#npz_train = data_path + '/price_history_03_dp_60to30_57980_train.npz'
#npz_train = data_path + '/price_history_03_dp_60to30_6400_train.npz'
#npz_train = data_path + '/price_history_60to30_6400_targets_normed_train.npz'
npz_train = data_path + '/price_history_03_dp_60to30_6400_global_remove_scale_train.npz'

#npz_test = data_path + '/price_history_03_dp_60to30_test.npz'
#npz_test = data_path + '/price_history_60to30_targets_normed_test.npz'
npz_test = data_path + '/price_history_03_dp_60to30_global_remove_scale_test.npz'

#%%time
# csv_in = '../price_history_03_seq_start_suddens_trimmed.csv'

# train_sku_ids, train_XX, train_YY, train_sequence_lens, train_seq_mask, test_pack = \
#     PriceHistoryDatasetGenerator(random_state=random_state).createAndSaveDataset(
#             csv_in=csv_in,
#             input_seq_len=input_len,
#             target_seq_len=target_len,
#             allowSmallerSequencesThanWindow=False,
#             #min_date = '2016-11-01',
#             split_fraction = 0.40,
#             #keep_training_fraction = 0.22, #57994 * 0.22 = 12758.68
#             normalize_targets = False,
#             do_global_norm_scale = True,
#             #disable saving for now since we have already created them
#             save_files_dic = {"train": npz_full_train, "test": npz_test,},
#     )

# print train_sku_ids.shape, train_XX.shape, train_YY.shape, train_sequence_lens.shape, train_seq_mask.shape
# aa,bb,cc,dd,ee = test_pack.get_data()
# aa.shape,bb.shape,cc.shape,dd.shape,ee.shape

# PriceHistoryDatasetGenerator.create_subsampled(inpath=npz_trainfull_train,
#                                                target_size = train_size,
#                                                outpath = npz_train, random_state=random_state)


I am working on the file "04_time_series_prediction/22_price_history_seq2seq-removed_scale.ipynb".
The first task is:
Recall that without batch normalization within 10 epochs with num units 400 and batch_size 64 we reached at 4.940
and with having the decoder inputs NOT filled from the outputs
Can you write Python code for it?

%%time
dyn_stats, preds_dict = get_or_run_nn(experiment, filename='022_seq2seq_60to30_001')

dyn_stats.plotStats()
plt.show()

r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]

ind = np.argmin(r2_scores)
ind

reals = dp.targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]

aa = fastdtw(dp.targets[0], preds_dict[0])

import fastdtw as fst
targets = dp.targets[0]
preds = preds_dict[0]
ind = 0

print type(targets)
print type(targets[ind])
print targets.shape
print type(preds)
print type(preds[ind])
print preds.shape

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


%%time
dyn_stats, preds_dict = get_or_run_nn(experiment, filename='022_seq2seq_60to30_002')

dyn_stats.plotStats()
plt.show()

r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]

ind = np.argmin(r2_scores)
ind

reals = dp.targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


I am working on the file "04_time_series_prediction/22_price_history_seq2seq-removed_scale.ipynb".
The first task is:
Step 1 - collect data
Can you write Python code for it?

dp = PriceHistorySeq2SeqDataProvider(npz_path=npz_train, batch_size=batch_size, with_EOS=with_EOS)
dp.inputs.shape, dp.targets.shape

aa, bb = dp.next()
aa.shape, bb.shape


I am working on the file "04_time_series_prediction/22_price_history_seq2seq-removed_scale.ipynb".
The first task is:
Step 2 - Build model
Can you write Python code for it?

model = PriceHistorySeq2SeqDynDecIns(rng=random_state, dtype=dtype, config=config, with_EOS=with_EOS)

# graph = model.getGraph(batch_size=batch_size,
#                        num_units=num_units,
#                        input_len=input_len,
#                        target_len=target_len)

#show_graph(graph)


I am working on the file "04_time_series_prediction/22_price_history_seq2seq-removed_scale.ipynb".
The first task is:
Step 3 training the network
RECALL: baseline is around 4 for huber loss for current problem, anything above 4 should be considered as major errors
Can you write Python code for it?

def experiment():
    return model.run(npz_path=npz_train,
             epochs=20,
             batch_size = 50,
             num_units = 400,
             input_len=input_len,
             target_len=target_len,
             #learning_rate = 0.0026945952539362472,
             preds_gather_enabled=True,
             batch_norm_enabled = True,
             activation = tf.nn.elu,
             decoder_first_input = PriceHistorySeq2SeqDynDecIns.DECODER_FIRST_INPUT.ZEROS,
             #keep_prob_input = 0.9,
             #lamda2 = 1e-2,
            )


def experiment():
    return model.run(npz_path=npz_train,
             epochs=20,
             batch_size = 50,
             num_units = 400,
             input_len=input_len,
             target_len=target_len,
             #learning_rate = 0.0026945952539362472,
             preds_gather_enabled=True,
             batch_norm_enabled = True,
             activation = tf.nn.elu,
             decoder_first_input = PriceHistorySeq2SeqDynDecIns.DECODER_FIRST_INPUT.PREVIOUS_INPUT,
             #keep_prob_input = 0.9,
             #lamda2 = 1e-2,
            )


I am working on the file "04_time_series_prediction/23_price_history_seq2seq-cross-validation.ipynb".
The first task is:
Cross Validating
Can you write Python code for it?

def plotter(stats_list, label_text):
    _ = renderStatsListWithLabels(stats_list=stats_list, label_text=label_text)
    plt.show()

    _ = renderStatsListWithLabels(stats_list=stats_list, label_text=label_text,
                                  title='Validation Error', kk='error(valid)')
    plt.show()

#sorted(factors(6400))

obj = PriceHistory23GpOpt(model=model,
                          stats_npy_filename = 'bayes_opt_23_stats_dic',
                          cv_score_dict_npy_filename = 'bayes_opt_23_cv_scores_dic',
                          random_state=random_state,
                          plotter = plotter,
                          npz_path=npz_train,
                          epochs=15,
                          batch_size=batch_size,
                          input_len=input_len,
                          target_len=target_len,
                          n_splits=5,
                         )

opt_res = obj.run_opt(n_random_starts=2, n_calls=17)

plot_res_gp(opt_res)

opt_res.best_params

filepath = PriceHistory23GpOpt.bayes_opt_dir + '/bayes_opt_23_stats_dic.npy'

renderStatsCollectionOfCrossValids(stats_dic=np.load(filepath)[()], label_texts=[
    'num_units', 'activation', 'lamda2', 'keep_prob_input', 'learning_rate'])
plt.show()


I am working on the file "04_time_series_prediction/23_price_history_seq2seq-cross-validation.ipynb".
The first task is:
Step 1 - collect data
Can you write Python code for it?

# dp = PriceHistorySeq2SeqDataProvider(npz_path=npz_train, batch_size=batch_size, with_EOS=with_EOS)
# dp.inputs.shape, dp.targets.shape

# aa, bb = dp.next()
# aa.shape, bb.shape


I am working on the file "04_time_series_prediction/23_price_history_seq2seq-cross-validation.ipynb".
The first task is:
Step 3 training the network
Can you write Python code for it?

model = PriceHistorySeq2SeqDynDecIns(rng=random_state, dtype=dtype, config=config, with_EOS=with_EOS)

opt_res.best_params

num_units, activation, lamda2, keep_prob_input, learning_rate = opt_res.best_params

batch_size

npz_1280_test = '../data/price_history/price_history_03_dp_60to30_global_remove_scale_targets_normed_1280_test.npz'

# PriceHistoryDatasetGenerator.create_subsampled(inpath=npz_test, target_size=1280,
#     outpath='../data/price_history/price_history_03_dp_60to30_global_remove_scale_targets_normed_1280_test.npz',
#                                               random_state=random_state)

def experiment():
    return model.run(npz_path=npz_train,
             npz_test = npz_1280_test,
             epochs=200,
             batch_size = batch_size,
             num_units = num_units,
             input_len=input_len,
             target_len=target_len,
             learning_rate = learning_rate * 10,
             preds_gather_enabled=True,
             batch_norm_enabled = True,
             activation = activation,
             decoder_first_input = PriceHistorySeq2SeqDynDecIns.DECODER_FIRST_INPUT.ZEROS,
             keep_prob_input = keep_prob_input,
             lamda2 = lamda2,
            )

#%%time
dyn_stats, preds_dict, targets = get_or_run_nn(experiment, filename='023_seq2seq_60to30_001')

dyn_stats.plotStats()
plt.show()

r2_scores = [r2_score(y_true=targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(targets))]

ind = np.argmin(r2_scores)
ind

reals = targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

#sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(targets[ind], preds_dict[ind])[0]
             for ind in range(len(targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(targets))
reals = targets[cur_ind]
preds = preds_dict[cur_ind]
fig =...


I am working on the file "04_time_series_prediction/23_price_history_seq2seq-cross-validation.ipynb".
The first task is:
https://www.youtube.com/watch?v=ElmBrKyMXxs
https://github.com/hans/ipython-notebooks/blob/master/tf/TF%20tutorial.ipynb
https://github.com/ematvey/tensorflow-seq2seq-tutorials
Can you write Python code for it?

from __future__ import division
import tensorflow as tf
from os import path, remove
import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import StratifiedShuffleSplit
from time import time
from matplotlib import pyplot as plt
import seaborn as sns
from mylibs.jupyter_notebook_helper import show_graph, renderStatsList, renderStatsCollection, \
    renderStatsListWithLabels, renderStatsCollectionOfCrossValids, plot_res_gp, my_plot_convergence
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
import shutil
from tensorflow.contrib.learn.python.learn import learn_runner
from mylibs.tf_helper import getDefaultGPUconfig
from sklearn.metrics import r2_score
from mylibs.py_helper import factors
from fastdtw import fastdtw
from collections import OrderedDict
from scipy.spatial.distance import euclidean
from statsmodels.tsa.stattools import coint
from common import get_or_run_nn
from data_providers.price_history_seq2seq_data_provider import PriceHistorySeq2SeqDataProvider
from data_providers.price_history_dataset_generator import PriceHistoryDatasetGenerator
from skopt.space.space import Integer, Real
from skopt import gp_minimize
from skopt.plots import plot_convergence
import pickle
import inspect
import dill
import sys
from models.price_history_21_seq2seq_dyn_dec_ins import PriceHistorySeq2SeqDynDecIns
from gp_opt.price_history_23_gp_opt import PriceHistory23GpOpt

dtype = tf.float32
seed = 16011984
random_state = np.random.RandomState(seed=seed)
config = getDefaultGPUconfig()
n_jobs = 1
%matplotlib inline


I am working on the file "04_time_series_prediction/24_price_history_seq2seq-full_dataset_testing.ipynb".
The first task is:
Once generate data
Can you write Python code for it?

data_folder = '../../../../Dropbox/data'

ph_data_path = '../data/price_history'

npz_full = ph_data_path + '/price_history_dp_60to30_57994.npz'
npz_train = ph_data_path + '/price_history_dp_60to30_57994_46400_train.npz'
npz_test = ph_data_path + '/price_history_dp_60to30_57994_11584_test.npz'


I am working on the file "04_time_series_prediction/24_price_history_seq2seq-full_dataset_testing.ipynb".
The first task is:
One epoch takes approximately 268 secs
If we want to let it run for ~8 hours = 8 * 3600 / 268 ~= 107 epochs
So let it run for 100 epochs and see how it behaves
Can you write Python code for it?

dyn_stats.plotStats()
plt.show()

data_len = len(targets)

mses = np.empty(data_len)
for ii, (pred, target) in enumerate(zip(preds_dict.values(), targets.values())):
    mses[ii] = mean_squared_error(pred, target)

np.mean(mses)

huber_losses = np.empty(data_len)
for ii, (pred, target) in enumerate(zip(preds_dict.values(), targets.values())):
    huber_losses[ii] = np.mean(huber_loss(pred, target))

np.mean(huber_losses)

targets_arr = np.array(targets.values())
targets_arr.shape

preds_arr = np.array(preds_dict.values())
preds_arr.shape

np.mean(huber_loss(y_true=targets_arr, y_pred=preds_arr))

r2_scores = [r2_score(y_true=targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(targets))]

ind = np.argmin(r2_scores)
ind

reals = targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

#sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(targets[ind], preds_dict[ind])[0]
             for ind in range(len(targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(targets))
reals = targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


I am working on the file "04_time_series_prediction/24_price_history_seq2seq-full_dataset_testing.ipynb".
The first task is:
Step 1 - collect data
Can you write Python code for it?


I am working on the file "04_time_series_prediction/24_price_history_seq2seq-full_dataset_testing.ipynb".
The first task is:
Step 2 - Build model
Can you write Python code for it?


I am working on the file "04_time_series_prediction/24_price_history_seq2seq-full_dataset_testing.ipynb".
The first task is:
Step 3 training the network
Can you write Python code for it?

best_params = [500,
 tf.nn.tanh,
 0.0001,
 0.62488034788862112,
 0.001]

num_units, activation, lamda2, keep_prob_input, learning_rate = best_params

batch_size

def experiment():
    return model.run(npz_path=npz_train,
             npz_test = npz_test,
             epochs=100,
             batch_size = batch_size,
             num_units = num_units,
             input_len=input_len,
             target_len=target_len,
             learning_rate = learning_rate,
             preds_gather_enabled=True,
             batch_norm_enabled = True,
             activation = activation,
             decoder_first_input = PriceHistorySeq2SeqDynDecIns.DECODER_FIRST_INPUT.ZEROS,
             keep_prob_input = keep_prob_input,
             lamda2 = lamda2,
            )

#%%time
dyn_stats, preds_dict, targets = get_or_run_nn(experiment, filename='024_seq2seq_60to30_002',
                                               nn_runs_folder= data_folder + '/nn_runs')


I am working on the file "04_time_series_prediction/25_price_history_plus_mobile_attributes.ipynb".
The first task is:
Step 0 - hyperparams
vocab_size is all the potential words you could have (classification for translation case)
and max sequence length are the SAME thing
decoder RNN hidden units are usually same size as encoder RNN hidden units in translation but for our case it does not seem really to be a relationship there but we can experiment and find out later, not a priority thing right now
Can you write Python code for it?

num_units = 400 #state size

input_len = 60
target_len = 30

batch_size = 50
with_EOS = False

total_train_size = 57994


I am working on the file "04_time_series_prediction/26_price_history_generate_train_test_and_baseline.ipynb".
The first task is:
Baseline is static, a straight line for each input - Test
Can you write Python code for it?

bltest = MyBaseline(npz_path=npz_test)

bltest.getMSE()

bltest.renderMSEs()
plt.show()

bltest.getHuberLoss()

bltest.renderHuberLosses()
plt.show()

bltest.get_dtw()

bltest.renderRandomTargetVsPrediction()
plt.show()


I am working on the file "04_time_series_prediction/26_price_history_generate_train_test_and_baseline.ipynb".
The first task is:
Baseline is static, a straight line for each input - Train (reduced)
Can you write Python code for it?

cur_baseline = MyBaseline(npz_path=npz_train_reduced)

cur_baseline.getMSE()

cur_baseline.renderMSEs()
plt.show()

cur_baseline.getHuberLoss()

cur_baseline.renderHuberLosses()
plt.show()

cur_baseline.get_dtw()

cur_baseline.renderRandomTargetVsPrediction()
plt.show()


I am working on the file "04_time_series_prediction/26_price_history_generate_train_test_and_baseline.ipynb".
The first task is:
Baseline is static, a straight line for each input - Train Full
Can you write Python code for it?

cur_baseline = MyBaseline(npz_path=npz_train)

cur_baseline.getMSE()

cur_baseline.renderMSEs()
plt.show()

cur_baseline.getHuberLoss()

cur_baseline.renderHuberLosses()
plt.show()

cur_baseline.get_dtw()

cur_baseline.renderRandomTargetVsPrediction()
plt.show()


I am working on the file "04_time_series_prediction/27_price_history_generate_train_test_and_baseline.ipynb".
The first task is:
Baseline is static, a straight line for each input - Train (small)
Can you write Python code for it?

bltrainsmall = MyBaseline(npz_path=npz_train_reduced)

bltrainsmall.getMSE()

bltrainsmall.renderMSEs()
plt.show()

bltrainsmall.getHuberLoss()

bltrainsmall.renderHuberLosses()
plt.show()

# myran = lambda: np.random.randn() * 1e-3
# dtw_scores = [fastdtw(bltrainsmall.targets[ind], bltrainsmall.targets[ind] + myran())[0]
#               for ind in range(len(bltrainsmall.targets))]
# np.mean(dtw_scores)

%%time
bltrainsmall.get_dtw()

bltrainsmall.renderRandomTargetVsPrediction()
plt.show()


I am working on the file "04_time_series_prediction/27_price_history_mobattrs_date_generation.ipynb".
The first task is:
Step 0 - hyperparams
vocab_size is all the potential words you could have (classification for translation case)
and max sequence length are the SAME thing
decoder RNN hidden units are usually same size as encoder RNN hidden units in translation but for our case it does not seem really to be a relationship there but we can experiment and find out later, not a priority thing right now
Can you write Python code for it?


I am working on the file "04_time_series_prediction/28_price_history_mobattrs_date_relevant_deals_generation.ipynb".
The first task is:
Include relevant deals
Can you write Python code for it?

from time import sleep

data_path = '../../../../Dropbox/data'
ph_data_path = data_path + '/price_history'
npz_full = ph_data_path + '/price_history_mobattrs_date_dp_60to30_62020.npz'
assert path.isfile(npz_full)

csv_in = '../price_history_03_seq_start_suddens_trimmed.csv'

#npz_train = ph_data_path + '/price_history_dp_60to30_63548_46400_train.npz'
#npz_train_mobattrs = ph_data_path + '/price_history_mobattrs_dp_60to30_57994_train.npz'

# npz_test = ph_data_path + '/price_history_dp_60to30_57994_11584_test.npz'
# npz_test_mobattrs = ph_data_path + '/price_history_mobattrs_dp_60to30_57994_test.npz'

obj = PriceHistWithRelevantDeals(npz_path=npz_full, price_history_csv_path=csv_in, random_state=random_state,
                                verbose=False)

dic = obj.execute(relevancy_count=2)

npz_augmented = ph_data_path + '/price_history_mobattrs_date_deals_dp_60to30.npz'

dic.keys()

dic['inputs'][0].shape

for key, val in dic.iteritems():
    print key, len(val)

len(dic['targets'])

len(dic['inputs'])

args = np.argwhere([curin.shape != (60, 9) for curin in dic['inputs']]).flatten()
len(args)

args = list(args)

# for cur in dic['inputs'][args]:
#     print cur.shape
for arg in args:
    print dic['inputs'][arg].shape

keep_args = set(range(len(dic['inputs']))).difference(args)

assert len(keep_args) == len(dic['inputs']) - len(args)
keep_args = list(keep_args)

newdic = {
    'inputs': np.array([dic[key][keep_arg] for keep_arg in keep_args])
}
newdic['inputs'].shape

for key, val in dic.iteritems():
    if key == 'inputs':
        continue
    newdic[key] = dic[key][keep_args]
    print newdic[key].shape

#dic['inputs'] = np.array(dic['inputs'])
np.array(newdic['inputs']).shape

np.savez(npz_augmented, **newdic)

npz = np.load(npz_full)
for key, val in npz.iteritems():
    print key,val.shape

my_current_ind = 100 #just because

target_ind = npz['sku_ids'][my_current_ind]
target_ind #this is the SKU ID we are interested in now

from relevant_deals import RelevantDeals

rd = RelevantDeals()

all_deals =...


I am working on the file "04_time_series_prediction/28_price_history_mobattrs_date_relevant_deals_generation.ipynb".
The first task is:
So for the same date window if we find data from the relevant deal we are good to go
If we do NOT find them ... we could go and search the next deal within some limit of course
So if we exceed this limit, what could we do?

We could pass zeros is one solution
We could drop this item completely is another solution
Can you write Python code for it?

# for one sku id
cur_sku_id = relevant_sku_ids[0]
cur_sku_id

seq = PriceHistory27DatasetGenerator.extractSequence(df.loc[cur_sku_id])
len(seq)

check = seq.index[0] <= start_date and end_date <= seq.index[-1]
check

#extract the sequence of interest
begin_ind = np.argwhere(seq.index == start_date).flatten()[0]
begin_ind

ending_ind = np.argwhere(seq.index == end_date).flatten()[0]
ending_ind

seq_of_interest = seq[begin_ind:ending_ind+1]
seq_of_interest.shape

the_input.shape

sns.tsplot(seq_of_interest)
plt.show()

unbiased = PriceHistory27DatasetGenerator.removeBiasFromSeq(seq_of_interest)
sns.tsplot(unbiased)
plt.show()

ready_deal = unbiased.values[np.newaxis].T
ready_deal.shape

newinput = np.hstack((the_input, ready_deal))
newinput.shape

np.array([unbiased, unbiased]).T

aa = np.array([unbiased, unbiased]).T

ee = np.array([]).T
ee.shape

np.hstack((the_input, ee)).shape

mylist = []
for ii, jj in zip(range(-3, 0), range(0, 3)):
    mylist.append((ii, jj))

mylist

map(list, zip(*mylist))[1]

sns.tsplot(aa[:, 1])


I am working on the file "04_time_series_prediction/29_price_history_dataset_per_mobile_phone.ipynb".
The first task is:
Actual Run
Can you write Python code for it?

data_path = '../../../../Dropbox/data'
ph_data_path = data_path + '/price_history'
assert path.isdir(ph_data_path)

npz_full = ph_data_path + '/price_history_per_mobile_phone.npz'

dataset_gen = PriceHistoryDatasetPerMobilePhone(random_state=random_state)

dic = dataset_gen.genSaveDictionary(csv_in=csv_in, window_len=90, npz_out=npz_full)
dic.keys()


I am working on the file "04_time_series_prediction/29_price_history_dataset_per_mobile_phone.ipynb".
The first task is:
Arima
Can you write Python code for it?

from arima.arima_estimator import ArimaEstimator
import warnings
from collections import OrderedDict
from mylibs.py_helper import cartesian_coord
from arima.arima_cv import ArimaCV

parameters = OrderedDict([
    ('p_auto_regression_order', range(6)), #0-5
    ('d_integration_level', range(3)), #0-2
    ('q_moving_average', range(6)), #0-5
])

cart = cartesian_coord(*parameters.values())
cart.shape

cur_sku = dic.values()[0]
cur_sku.keys()

full_mat = cur_sku['train']
full_mat.shape

target_len

inputs = full_mat[:, :-target_len]
inputs.shape

targets = full_mat[:, -target_len:]
targets.shape

%%time
with warnings.catch_warnings():
    warnings.filterwarnings("ignore")
    ae = ArimaEstimator(p_auto_regression_order=0, d_integration_level=1, q_moving_average=0, easy_mode=True)
    ae.fit(inputs, targets).score(inputs, targets)

score_dic_filepath = data_path + "/arima/scoredic_testing.npy"

path.abspath(score_dic_filepath)

%%time
with warnings.catch_warnings():
    warnings.filterwarnings("ignore")
    scoredic = ArimaCV.cross_validate(inputs=inputs, targets=targets, cartesian_combinations=cart,
                                      score_dic_filepath=score_dic_filepath, easy_mode=True)

#4h 4min 51s / 108 cases => ~= 136 seconds per case !

arr = np.array(list(scoredic.iteritems()))
arr.shape

np.NaN

import math

float('nan') == np.NaN

#np.isnan()
filtered_arr = arr[ np.logical_not(arr[:, 1] != arr[:, 1]) ]
filtered_arr.shape

plt.plot(filtered_arr[:, 1])

minarg = np.argmin(filtered_arr[:, 1])
minarg

best_params = filtered_arr[minarg, 0]
best_params

test_mat = cur_sku['test']
test_ins = test_mat[:-target_len]
test_ins.shape

test_tars = test_mat[-target_len:]
test_tars.shape

test_ins_vals = test_ins.values.reshape(1, -1)
test_ins_vals.shape

test_tars_vals = test_tars.values.reshape(1, -1)
test_tars_vals.shape


I am working on the file "04_time_series_prediction/29_price_history_dataset_per_mobile_phone.ipynb".
The first task is:
Step 0 - hyperparams
vocab_size is all the potential words you could have (classification for translation case)
and max sequence length are the SAME thing
decoder RNN hidden units are usually same size as encoder RNN hidden units in translation but for our case it does not seem really to be a relationship there but we can experiment and find out later, not a priority thing right now
Can you write Python code for it?

input_len = 60
target_len = 30
batch_size = 50
with_EOS = False

csv_in = '../price_history_03_seq_start_suddens_trimmed.csv'


I am working on the file "04_time_series_prediction/29_price_history_dataset_per_mobile_phone.ipynb".
The first task is:
Testing with easy mode off
Can you write Python code for it?

%%time
with warnings.catch_warnings():
    warnings.filterwarnings("ignore")
    ae = ArimaEstimator(p_auto_regression_order=best_params[0],
                        d_integration_level=best_params[1],
                        q_moving_average=best_params[2],
                        easy_mode=False)
    score = ae.fit(test_ins_vals, test_tars_vals).score(test_ins_vals, test_tars_vals)

score

plt.figure(figsize=(15,7))
plt.plot(ae.preds.flatten(), label='preds')
test_tars.plot(label='real')
plt.legend()
plt.show()


I am working on the file "04_time_series_prediction/30_price_history_dataset_per_mobile_phone-arima.ipynb".
The first task is:
Arima
Can you write Python code for it?

parameters = OrderedDict([
    ('p_auto_regression_order', range(6)), #0-5
    ('d_integration_level', range(3)), #0-2
    ('q_moving_average', range(6)), #0-5
])

cart = cartesian_coord(*parameters.values())
cart.shape

cur_key = dic.keys()[0]
cur_key

cur_sku = dic[cur_key][()]
cur_sku.keys()

train_mat = cur_sku['train']
train_mat.shape

target_len

inputs = train_mat[:, :-target_len]
inputs.shape

targets = train_mat[:, -target_len:]
targets.shape

easy_mode = False

score_dic_filepath = data_path + "/arima/scoredic_easy_mode_{}_{}.npy".format(easy_mode, cur_key)
path.abspath(score_dic_filepath)

%%time
with warnings.catch_warnings():
    warnings.filterwarnings("ignore")
    scoredic = ArimaCV.cross_validate(inputs=inputs, targets=targets, cartesian_combinations=cart,
                                      score_dic_filepath=score_dic_filepath, easy_mode=easy_mode)

#4h 4min 51s / 108 cases => ~= 136 seconds per case !

arr = np.array(list(scoredic.iteritems()))
arr.shape

#np.isnan()
filtered_arr = arr[ np.logical_not(arr[:, 1] != arr[:, 1]) ]
filtered_arr.shape

plt.plot(filtered_arr[:, 1])

minarg = np.argmin(filtered_arr[:, 1])
minarg

best_params = filtered_arr[minarg, 0]
best_params

test_mat = cur_sku['test']
test_ins = test_mat[:-target_len]
test_ins.shape

test_tars = test_mat[-target_len:]
test_tars.shape

test_ins_vals = test_ins.values.reshape(1, -1)
test_ins_vals.shape

test_tars_vals = test_tars.values.reshape(1, -1)
test_tars_vals.shape


I am working on the file "04_time_series_prediction/30_price_history_dataset_per_mobile_phone-arima.ipynb".
The first task is:
Conclusion
If you are training in easy mode then what you get at the end is that the model only cares for the previous value in order to do its predictions and this makes it much easier for everybody but in reality we might not have advantage
Trying
Can you write Python code for it?

args = np.argsort(filtered_arr[:, 1])
args

filtered_arr[args[:10], 0]

%%time
with warnings.catch_warnings():
    warnings.filterwarnings("ignore")
    ae = ArimaEstimator(p_auto_regression_order=4,
                        d_integration_level=1,
                        q_moving_average=3,
                        easy_mode=False)
    print ae.fit(test_ins_vals, test_tars_vals).score(test_ins_vals, test_tars_vals)

plt.figure(figsize=(15,7))
plt.plot(ae.preds.flatten(), label='preds')
test_tars.plot(label='real')
plt.legend()
plt.show()


I am working on the file "04_time_series_prediction/30_price_history_dataset_per_mobile_phone-arima.ipynb".
The first task is:
Testing with easy mode off
Can you write Python code for it?


I am working on the file "04_time_series_prediction/30_price_history_dataset_per_mobile_phone-arima.ipynb".
The first task is:
Testing with easy mode on
Can you write Python code for it?

%%time
with warnings.catch_warnings():
    warnings.filterwarnings("ignore")
    ae = ArimaEstimator(p_auto_regression_order=best_params[0],
                        d_integration_level=best_params[1],
                        q_moving_average=best_params[2],
                        easy_mode=True)
    score = ae.fit(test_ins_vals, test_tars_vals).score(test_ins_vals, test_tars_vals)

score

plt.figure(figsize=(15,7))
plt.plot(ae.preds.flatten(), label='preds')
test_tars.plot(label='real')
plt.legend()
plt.show()


I am working on the file "04_time_series_prediction/31_arima_stationary_price_history.ipynb".
The first task is:

Can you write Python code for it?

parameters = OrderedDict([
    ('p_auto_regression_order', range(6)), #0-5
    ('d_integration_level', range(3)), #0-2
    ('q_moving_average', range(6)), #0-5
])

def cartesian_coord(*arrays):
    grid = np.meshgrid(*arrays)
    coord_list = [entry.ravel() for entry in grid]
    points = np.vstack(coord_list).T
    return points

cart = cartesian_coord(*parameters.values())
cart.shape

108 * 15 / 60 / 2 #in hours!

count_inds=100
random_state = np.random.RandomState(seed=16011984)

random_inds = random_state.choice(range(len(XX)), count_inds, replace=False)
xx = XX[random_inds]
yy = YY[random_inds]

filepath = 'scoredic.npy'

%%time
with warnings.catch_warnings():
    warnings.filterwarnings("ignore")
    for pp, dd, qq in cart:
        scoredic = np.load(filepath)[()] if path.isfile(filepath) else OrderedDict()
        cur_tuple = (pp, dd, qq)
        if cur_tuple in scoredic:
            continue
        else:
            ae = ArimaEstimator(p_auto_regression_order=pp, d_integration_level=dd, q_moving_average=qq,
                                easy_mode=True)
            scoredic[cur_tuple] = ae.fit(inputs=xx, targets=yy).score(xx, yy)
            np.save(filepath, scoredic)

from sklearn.model_selection import GridSearchCV

# Pick hyperparameters
# There is NO separation of training and testing set because there is no transfer of knowledge from the training
# that could be useful for validation
# Run Arima prediction steps for all instances of training dataset and get a fastdtw score from each case
# Get the mean of the fastdtw and this is the CV score
#
# Do this for all possible parameters

fitted = model.fit()

fitted.

model_fit = model.fit(disp=0)

model_fit.summary()

residuals = DataFrame(model_fit.resid)

residuals.plot()
plt.show()

residuals.plot(kind='kde')
plt.show()

residuals.describe()


I am working on the file "04_time_series_prediction/31_arima_stationary_price_history.ipynb".
The first task is:
Reverse Moving average
https://stats.stackexchange.com/questions/67907/extract-data-points-from-moving-average
Can you write Python code for it?

orig_seq.plot()

aa = np.zeros((orig_seq.size - window +1, orig_seq.size))
aa.shape

for ii, item in enumerate(aa):
    aa[ii, ii:ii+window] = 1/window

trans1 = aa.dot(orig_seq)
trans1.shape

from numpy.linalg import pinv

trans2 = orig_seq.rolling(window=window).mean()[window-1:].values
trans2.shape

np.allclose(trans1, trans2)

pA = pinv(aa)
pA.shape

reconstructed = pA.dot(trans1)
reconstructed.shape

plt.plot(reconstructed, label='reconstructed')
plt.plot(orig_seq.values, label='original')
plt.legend()
plt.show()

np.allclose(reconstructed, orig_seq.values, rtol=1e-3)

#np.sqrt(np.mean((reconstructed - orig_seq.values)**2)) #RMSE
np.corrcoef(orig_seq.values, reconstructed)


I am working on the file "04_time_series_prediction/31_arima_stationary_price_history.ipynb".
The first task is:
WTF What is going on ??
Can you write Python code for it?

np.corrcoef(orig_seq.values, custom_recons)

aa.shape, orig_seq.values.shape

trans4 = orig_seq.values[window-1:] - aa.dot(orig_seq.values)
np.allclose(trans4, seq.values)

bb = 1 - aa
bb.shape

trans3 = bb.dot(orig_seq.values)
trans3.shape

plt.plot(trans3)
#plt.plot(orig_seq.values)


I am working on the file "04_time_series_prediction/31_arima_stationary_price_history.ipynb".
The first task is:
http://machinelearningmastery.com/arima-for-time-series-forecasting-with-python/
http://ucanalytics.com/blogs/arima-models-manufacturing-case-study-example-part-3/
https://stats.stackexchange.com/questions/23036/estimating-same-model-over-multiple-time-series
The idea with ARIMA models is that the final residual should look like white noise otherwise, there is juice or information available in the data to extract
Can you write Python code for it?

from __future__ import division
from os import path, remove
import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import StratifiedShuffleSplit
from time import time
from matplotlib import pyplot as plt
import seaborn as sns
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
import shutil
from tensorflow.contrib.learn.python.learn import learn_runner
from sklearn.metrics import r2_score
from fastdtw import fastdtw
from collections import OrderedDict
from scipy.spatial.distance import euclidean
from statsmodels.tsa.stattools import coint
from skopt.space.space import Integer, Real
from skopt import gp_minimize
from skopt.plots import plot_convergence
import pickle
import inspect
import dill
import sys
from pandas import read_csv
from pandas import datetime
from pandas.plotting import autocorrelation_plot
from statsmodels.graphics.tsaplots import plot_pacf
from statsmodels.tsa.arima_model import ARIMA
from pandas import DataFrame
from sklearn.metrics import mean_squared_error
from price_history import PriceHistory
from arima.arima_estimator import ArimaEstimator
import warnings
from collections import OrderedDict

%matplotlib inline


I am working on the file "04_time_series_prediction/31_arima_stationary_price_history.ipynb".
The first task is:
p: The number of lag observations included in the model, also called the lag order.
d: The number of times that the raw observations are differenced, also called the degree of differencing.
q: The size of the moving average window, also called the order of moving average.
Can you write Python code for it?

data_path = '../../../../../Dropbox/data'

score_dic_filepath = data_path + "/arima/scoredic_non_stationary.npy"

ph_data_path = data_path + '/price_history'

csv_in = "../price_history_03_seq_start_suddens_trimmed.csv"

ph = PriceHistory(csv_in)

seqs = ph.extractAllSequences()

def custom_transformation(sequence, window):
    #logged_vals = np.log(sequence.values)
    logged_vals = sequence.values
    logged_seq = sequence.copy()
    logged_seq[logged_seq.index] = logged_vals
    return logged_seq - logged_seq.rolling(window=window).mean()

orig_seq = seqs[0]

window = 3

seq_tr = custom_transformation(sequence=seqs[0], window=window)

np.all(seq_tr == seq_tr)

seq = seq_tr[window-1:]

np.all(seq == seq)

plt.figure(figsize=(15,6))
seq.plot()
plt.show()

ae = ArimaEstimator(p_auto_regression_order=4, d_integration_level=1, q_moving_average=3, easy_mode=False)

target_len = 30

XX = seq[:-target_len].values.reshape((1, -1))
XX.shape

YY = seq[-target_len:].values.reshape((1, -1))
YY.shape

%%time
with warnings.catch_warnings():
    warnings.filterwarnings("ignore")
    ae.fit(inputs=XX, targets=YY)

preds = ae.preds.flatten()
yy = YY.flatten()
yy.shape, preds.shape

plt.figure(figsize=(15,6))
plt.plot(yy, label='real')
plt.plot(preds, label='preds')
plt.legend()
plt.show()


I am working on the file "04_time_series_prediction/32_price_history_autoencoder.ipynb".
The first task is:
(1, 682, 7)
(1, 682, 6)
Can you write Python code for it?

# for item in dp.next():
#     print item.shape


I am working on the file "04_time_series_prediction/32_price_history_autoencoder.ipynb".
The first task is:
Conclusion
For one instance that is the most easy case it seems to be trainable, let's get the predicted values to observe how it actually looks like
Step 3 training the network
Can you write Python code for it?

model = PriceHistoryAutoencoder(rng=random_state, dtype=dtype, config=config)

batch_size

npz_test = npz_one_instance_prefix + '_test.npz'
assert path.isfile(npz_test)
path.abspath(npz_test)

def experiment():
    return model.run(npz_path=npz_one_instance_prefix,
                     epochs=100,
                     batch_size = batch_size,
                     enc_num_units = 400,
                     dec_num_units = 400,
                     ts_len=210,
                     learning_rate = 1e-4,
                     preds_gather_enabled = True,
                    )

#%%time
dyn_stats, preds_dict, targets = get_or_run_nn(experiment, filename='032_autoencoder_000',
                                              nn_runs_folder = data_path + "/nn_runs")

dyn_stats.plotStats()
plt.show()

r2_scores = [r2_score(y_true=targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(targets))]

ind = np.argmin(r2_scores)
ind

reals = targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

#sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(targets[ind], preds_dict[ind])[0]
             for ind in range(len(targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(targets))
reals = targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b', label='reals')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


I am working on the file "04_time_series_prediction/32_price_history_autoencoder.ipynb".
The first task is:
Once generate data
Can you write Python code for it?

df = pd.read_csv(csv_in, index_col=0, quoting=csv.QUOTE_ALL, encoding='utf-8')
df.shape

npz_path = phae_path + '/price_history_full_seqs.npz'

%%time
train_pack, test_pack = PriceHistoryAutoEncoderDatasetGenerator(random_state=random_state).createAndSaveDataset(
    csv_in = csv_in,
    do_global_norm_scale = True,
    save_files_dic = {
        'train': npz_path,
        'test': None,
    }
)

#sku_ids
#inputs
#sequence lengths
#masks
#dates
for bb in train_pack.get_data():
    print bb.shape

%%time
merged_dic = PriceHistoryAutoEncoderDatasetGenerator.merge_date_info(npz_path=npz_path)

max_seq_len = merged_dic['inputs'].shape[1]
max_seq_len

assert len(np.argwhere(merged_dic['extra_inputs'][0][:, 0] == -1).flatten()) + merged_dic['sequence_lengths'][0] \
    == max_seq_len, "just checking if our conversion occured as it should have"

npz_dates = phae_path + '/price_history_full_seqs_dates.npz'
#np.savez(npz_dates, **merged_dic)

dic = np.load(npz_dates)
for key, val in dic.iteritems():
    print key, ",", val.shape

years = dic['extra_inputs'][:, :, 3]
years.shape

years_flat = years.flatten()
years_flat.shape

years_filtered = years_flat[years_flat >= 0]
years_filtered.shape

np.mean(years_filtered)

np.std(years_filtered)

%%time
norm_dates_dic = PriceHistoryAutoEncoderDatasetGenerator.normalize_date_info(npz_dates)
for key, val in norm_dates_dic.iteritems():
    print key, ",", val.shape

norm_dates_dic['extra_inputs'][:, :, 0]

npz_dates = phae_path + '/price_history_seqs_dates_normed_train.npz'
np.savez(npz_dates, **norm_dates_dic)

final_dic = np.load(npz_dates)

for key, val in final_dic.iteritems():
    print key, ",", val.shape

small_dic = {}
for key, val in final_dic.iteritems():
    small_dic[key] = val[:1]
    print key, ",", small_dic[key].shape

np.savez(npz_one_instance, **small_dic)

# PriceHistoryDatasetGenerator.create_subsampled(inpath=npz_train_full, target_size=55800,
#                                                outpath=npz_train_trimmed, random_state=random_state)


I am working on the file "04_time_series_prediction/32_price_history_autoencoder.ipynb".
The first task is:
Quick test run
Can you write Python code for it?

def experiment():
    return model.run(npz_path=npz_one_instance_prefix,
                     epochs=100,
                     batch_size = batch_size,
                     enc_num_units = 400,
                     dec_num_units = 400,
                     ts_len=210,
                     learning_rate = 1e-4,
                     preds_gather_enabled = False,
                    )
dyn_stats = experiment()

dyn_stats.plotStats()


I am working on the file "04_time_series_prediction/32_price_history_autoencoder.ipynb".
The first task is:
Step 0 - hyperparams
vocab_size is all the potential words you could have (classification for translation case)
and max sequence length are the SAME thing
decoder RNN hidden units are usually same size as encoder RNN hidden units in translation but for our case it does not seem really to be a relationship there but we can experiment and find out later, not a priority thing right now
Can you write Python code for it?

batch_size = 1

#full_train_size = 55820
#train_size = 55800
#small_train_size = 6000 #just because of performance reasons, no statistics behind this decision
#test_size = 6200

data_path = '../../../../Dropbox/data'

phae_path = data_path + '/price_hist_autoencoder'

csv_in = '../price_history_03_seq_start_suddens_trimmed.csv'
assert path.isfile(csv_in)

npz_one_instance = phae_path + '/price_history_seqs_dates_normed_one_instance_train.npz'

npz_one_instance_prefix = npz_one_instance[:-len('_train.npz')]


I am working on the file "04_time_series_prediction/32_price_history_autoencoder.ipynb".
The first task is:
Step 2 - Build model
Can you write Python code for it?

model = PriceHistoryAutoencoder(rng=random_state, dtype=dtype, config=config)

# graph = model.getGraph(batch_size=batch_size,
#                        enc_num_units = 10,
#                        dec_num_units = 10,
#                        ts_len=210)


I am working on the file "04_time_series_prediction/32_price_history_autoencoder.ipynb".
The first task is:
Tensor("data/strided_slice:0", shape=(50, 210), dtype=float32)
210
Tensor("inputs/unstack:0", shape=(50, 7), dtype=float32)
Tensor("encoder_rnn_layer/rnn/gru_cell_209/add:0", shape=(50, 10), dtype=float32)
Tensor("encoder_state_out_process/Elu:0", shape=(50, 2), dtype=float32)
Tensor("decoder_state_in_process/Elu:0", shape=(50, 10), dtype=float32)
210
Tensor("dec_extra_ins/unstack:0", shape=(50, 6), dtype=float32)
decoder_outputs len: 210
Tensor("decoder_rnn_layer/rnn/gru_cell/add:0", shape=(50, 10), dtype=float32)
Tensor("decoder_outs/stack:0", shape=(50, 210, 10), dtype=float32)
Tensor("decoder_outs/Reshape:0", shape=(10500, 10), dtype=float32)
Tensor("readout_affine/Identity:0", shape=(10500, 1), dtype=float32)
Tensor("readout_affine/Reshape:0", shape=(50, 210), dtype=float32)
Tensor("error/Select:0", shape=(50, 210), dtype=float32)
Tensor("error/Mean:0", shape=(), dtype=float32)
Tensor("error/Mean:0", shape=(), dtype=float32)
Can you write Python code for it?


I am working on the file "04_time_series_prediction/35_price_history_autoencoder_dyn_rnn_with_diff.ipynb".
The first task is:
Step 1 - collect data
Can you write Python code for it?

dp = PriceHistoryAutoEncDataProvider(npz_path=npz_path, batch_size=53, with_EOS=False)
for data in dp.datalist:
    print data.shape

# for item in dp.next():
#     print item.shape


I am working on the file "04_time_series_prediction/35_price_history_autoencoder_dyn_rnn_with_diff.ipynb".
The first task is:
targets
Tensor("data/strided_slice:0", shape=(53, 682), dtype=float32)
Tensor("encoder_rnn_layer/rnn/while/Exit_2:0", shape=(?, 10), dtype=float32)
Tensor("encoder_state_out_process/Elu:0", shape=(?, 2), dtype=float32)
Tensor("decoder_state_in_process/Elu:0", shape=(?, 10), dtype=float32)
Tensor("decoder_rnn_layer/rnn/transpose:0", shape=(53, 682, 10), dtype=float32)
Tensor("decoder_outs/Reshape:0", shape=(36146, 10), dtype=float32)
Tensor("readout_affine/Identity:0", shape=(36146, 1), dtype=float32)
Tensor("readout_affine/Reshape:0", shape=(53, 682), dtype=float32)
Tensor("error/mul_1:0", shape=(53, 682), dtype=float32)
Tensor("error/Mean:0", shape=(), dtype=float32)
Tensor("error/Mean:0", shape=(), dtype=float32)
Tensor("error_diff/mul_1:0", shape=(53, 681), dtype=float32)
Tensor("error_diff/Mean:0", shape=(), dtype=float32)
Tensor("error_diff/Mean:0", shape=(), dtype=float32)
Can you write Python code for it?


I am working on the file "04_time_series_prediction/35a_price_history_autoencoder_dyn_rnn_with_diff.ipynb".
The first task is:
Conclusion
The autoencoder is not able to represent in a visibly obvious way our price history time series
TS in Two Dimensions
Can you write Python code for it?

twod_arr = np.array(twods.values())
twod_arr.shape

plt.figure(figsize=(16,7))
plt.plot(twod_arr[:, 0], twod_arr[:, 1], 'r.')
plt.title('two dimensional representation of our time series after dimensionality reduction')
plt.xlabel('first dimension')
plt.ylabel('second dimension')
plt.show()


I am working on the file "04_time_series_prediction/35a_price_history_autoencoder_dyn_rnn_with_diff.ipynb".
The first task is:
Quick test run
Can you write Python code for it?

def experiment():
    return model.run(npz_path=npz_path,
                     epochs=2,
                     batch_size = 53,
                     enc_num_units = 400,
                     dec_num_units = 400,
                     ts_len=max_seq_len,
                     learning_rate = 1e-4,
                     preds_gather_enabled = False,
                    )
dyn_stats_dic = experiment()

dyn_stats_dic['dyn_stats'].plotStats()
plt.show()
dyn_stats_dic['dyn_stats_diff'].plotStats()
plt.show()


I am working on the file "04_time_series_prediction/35a_price_history_autoencoder_dyn_rnn_with_diff.ipynb".
The first task is:
Step 0 - hyperparams
Can you write Python code for it?

factors(689)

max_seq_len = 682

#full_train_size = 55820
#train_size = 55800
#small_train_size = 6000 #just because of performance reasons, no statistics behind this decision
#test_size = 6200

data_path = '../../../../Dropbox/data'

phae_path = data_path + '/price_hist_autoencoder'

csv_in = '../price_history_03_seq_start_suddens_trimmed.csv'
assert path.isfile(csv_in)

npz_unprocessed = phae_path + '/price_history_full_seqs.npz'
assert path.isfile(npz_unprocessed)

npz_dates = phae_path + '/price_history_full_seqs_dates.npz'
assert path.isfile(npz_dates)

npz_train = phae_path + '/price_history_seqs_dates_normed_train.npz'
assert path.isfile(npz_train)

npz_test = phae_path + '/price_history_seqs_dates_normed_test.npz'
assert path.isfile(npz_test)

npz_path = npz_train[:-len('_train.npz')]

for key, val in np.load(npz_train).iteritems():
    print key, ",", val.shape


I am working on the file "04_time_series_prediction/35a_price_history_autoencoder_dyn_rnn_with_diff.ipynb".
The first task is:
Step 1 - collect data
Can you write Python code for it?


I am working on the file "04_time_series_prediction/35a_price_history_autoencoder_dyn_rnn_with_diff.ipynb".
The first task is:
Step 2 - Build model
Can you write Python code for it?

# model = PriceHistoryAutoencoder(rng=random_state, dtype=dtype, config=config)
# graph = model.getGraph(batch_size=53,
#                        enc_num_units = 10,
#                        dec_num_units = 10,
#                        ts_len=max_seq_len)


I am working on the file "04_time_series_prediction/35a_price_history_autoencoder_dyn_rnn_with_diff.ipynb".
The first task is:
Step 3 training the network
Can you write Python code for it?

model = PriceHistoryAutoencoder(rng=random_state, dtype=dtype, config=config)

npz_test = npz_path + '_test.npz'
assert path.isfile(npz_test)
path.abspath(npz_test)

def experiment():
    return model.run(npz_path=npz_path,
                     epochs=50,
                     batch_size = 53,
                     enc_num_units = 450,
                     dec_num_units = 450,
                     ts_len=max_seq_len,
                     learning_rate = 1e-3,
                     preds_gather_enabled = True,
                    )

#%%time
# dyn_stats_dic, preds_dict, targets, twods = experiment()
dyn_stats_dic, preds_dict, targets, twods = get_or_run_nn(experiment, filename='035_autoencoder_001',
                                                      nn_runs_folder = data_path + "/nn_runs")

dyn_stats_dic['dyn_stats'].plotStats()
plt.show()
dyn_stats_dic['dyn_stats_diff'].plotStats()
plt.show()

r2_scores = [r2_score(y_true=targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(targets))]

ind = np.argmin(r2_scores)
ind

reals = targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

#sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(targets[ind], preds_dict[ind])[0]
             for ind in range(len(targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(targets))
reals = targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b', label='reals')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


I am working on the file "04_time_series_prediction/35a_price_history_autoencoder_dyn_rnn_with_diff.ipynb".
The first task is:
targets
Tensor("data/strided_slice:0", shape=(53, 682), dtype=float32)
Tensor("encoder_rnn_layer/rnn/while/Exit_2:0", shape=(?, 10), dtype=float32)
Tensor("encoder_state_out_process/Elu:0", shape=(?, 2), dtype=float32)
Tensor("decoder_state_in_process/Elu:0", shape=(?, 10), dtype=float32)
Tensor("decoder_rnn_layer/rnn/transpose:0", shape=(53, 682, 10), dtype=float32)
Tensor("decoder_outs/Reshape:0", shape=(36146, 10), dtype=float32)
Tensor("readout_affine/Identity:0", shape=(36146, 1), dtype=float32)
Tensor("readout_affine/Reshape:0", shape=(53, 682), dtype=float32)
Tensor("error/mul_1:0", shape=(53, 682), dtype=float32)
Tensor("error/Mean:0", shape=(), dtype=float32)
Tensor("error/Mean:0", shape=(), dtype=float32)
Tensor("error_diff/mul_1:0", shape=(53, 681), dtype=float32)
Tensor("error_diff/Mean:0", shape=(), dtype=float32)
Tensor("error_diff/Mean:0", shape=(), dtype=float32)
Can you write Python code for it?


I am working on the file "04_time_series_prediction/37_price_history_clustering.ipynb".
The first task is:
(689, 682, 7)
(689, 682, 6)
(689,)
(689, 682)
Can you write Python code for it?


I am working on the file "04_time_series_prediction/37_price_history_clustering.ipynb".
The first task is:
Conclusion
Visually it SEEMS meaningful to have six(6) clusters. But this is a very big if. However this will help us segment our dataset a little bit and actual have the overhead that we are training six(6) different models!
If we want to split them further then 13 groups seems good enough
Storing Groups of SKU ids
Can you write Python code for it?

chosen_kk = 10

labels = KMeans(init='random', n_clusters=chosen_kk, n_init=100).fit(twod_arr).labels_

labels.shape

assert np.all(preds_dict.keys() == np.arange(len(preds_dict))), \
    "just make sure that the order of the predictions and other outputs are in order"
#since order is not shuffled this simplifies things

assert path.isfile(npz_test)

sku_ids = np.load(npz_test)['sku_ids']
sku_ids.shape

sku_dic = OrderedDict()
for ii in range(chosen_kk):
    sku_dic[str(ii)] = sku_ids[labels == ii]

assert np.sum([len(val) for val in sku_dic.values()]) == len(sku_ids), "weak check to ensure every item is assigned"

sku_ids_groups_path = data_path + '/sku_ids_groups'
assert path.isdir(sku_ids_groups_path)
path.abspath(sku_ids_groups_path)

sku_dic.keys()

for key, val in sku_dic.iteritems():
    print key, ",", val.shape

npz_sku_ids_group_kmeans = sku_ids_groups_path + '/sku_ids_kmeans_{:02d}.npz'.format(chosen_kk)
np.savez(npz_sku_ids_group_kmeans, **sku_dic)


I am working on the file "04_time_series_prediction/37_price_history_clustering.ipynb".
The first task is:
K-means clustering
Can you write Python code for it?

from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score

km = KMeans(init='random', #init='k-means++',
            n_clusters=3, n_init=10)

km.fit(twod_arr)

km.inertia_

silhouette_score(twod_arr, km.labels_)

def getKmeansScores():
    for kk in range(2, 20):
        km = KMeans(init='random', #init='k-means++',
                n_clusters=kk, n_init=100).fit(twod_arr)
        score = silhouette_score(twod_arr, km.labels_)
        yield kk, score

%%time
dic = OrderedDict(list(getKmeansScores()))

plt.bar(dic.keys(), dic.values())
plt.xticks(dic.keys())
plt.show()

from mylibs.kde_2d_label import kde_2d_label

km = KMeans(init='random', #init='k-means++',
                n_clusters=2, n_init=100).fit(twod_arr)
kde_2d_label(twod_arr, km.labels_)

kde_2d_label(twod_arr, KMeans(init='random', n_clusters=3, n_init=100).fit(twod_arr).labels_)

kde_2d_label(twod_arr, KMeans(init='random', n_clusters=4, n_init=100).fit(twod_arr).labels_)

kde_2d_label(twod_arr, KMeans(init='random', n_clusters=5, n_init=100).fit(twod_arr).labels_)

kde_2d_label(twod_arr, KMeans(init='random', n_clusters=6, n_init=100).fit(twod_arr).labels_)

kde_2d_label(twod_arr, KMeans(init='random', n_clusters=7, n_init=100).fit(twod_arr).labels_)

kde_2d_label(twod_arr, KMeans(init='random', n_clusters=8, n_init=100).fit(twod_arr).labels_)

kde_2d_label(twod_arr, KMeans(init='random', n_clusters=13, n_init=100).fit(twod_arr).labels_)


I am working on the file "04_time_series_prediction/37_price_history_clustering.ipynb".
The first task is:
Step 0 - hyperparams
Can you write Python code for it?

factors(689)

max_seq_len = 682

data_path = '../../../../Dropbox/data'

phae_path = data_path + '/price_hist_autoencoder'

npz_dates = phae_path + '/price_history_full_seqs_dates.npz'
assert path.isfile(npz_dates)

npz_train = phae_path + '/price_history_seqs_dates_normed_train.npz'
assert path.isfile(npz_train)

npz_test = phae_path + '/price_history_seqs_dates_normed_test.npz'
assert path.isfile(npz_test)

npz_path = npz_train[:-len('_train.npz')]

for key, val in np.load(npz_train).iteritems():
    print key, ",", val.shape


I am working on the file "04_time_series_prediction/37_price_history_clustering.ipynb".
The first task is:
Step 1 - collect data
Can you write Python code for it?

# dp = PriceHistoryAutoEncDataProvider(npz_path=npz_path, batch_size=53, with_EOS=False)
# for data in dp.datalist:
#     print data.shape


I am working on the file "04_time_series_prediction/37_price_history_clustering.ipynb".
The first task is:
Step 2 - Build model
Can you write Python code for it?

# model = PriceHistoryAutoencoder(rng=random_state, dtype=dtype, config=config)
# graph = model.getGraph(batch_size=53,

#                        #the way we have it these two must be equal for now
#                        enc_num_units = 10,
#                        hidden_enc_num_units = 10,

#                        hidden_enc_dim = 12,
#                        hidden_dec_dim = 13,

#                        #the way we have it these two must be equal for now
#                        hidden_dec_num_units = 14,
#                        dec_num_units = 14,

#                        ts_len=max_seq_len)


I am working on the file "04_time_series_prediction/37_price_history_clustering.ipynb".
The first task is:
targets
Tensor("data/strided_slice:0", shape=(53, 682), dtype=float32)
Tensor("encoder_rnn_layer/rnn/transpose:0", shape=(53, 682, 10), dtype=float32)
Tensor("encoder_rnn_layer/rnn/while/Exit_2:0", shape=(?, 10), dtype=float32)
Tensor("hidden_encoder_rnn_layer/rnn/transpose:0", shape=(53, 682, 10), dtype=float32)
Tensor("hidden_encoder_rnn_layer/rnn/while/Exit_2:0", shape=(?, 10), dtype=float32)
Tensor("encoder_state_out_hidden_process/Elu:0", shape=(?, 12), dtype=float32)
Tensor("encoder_state_out_process/Elu:0", shape=(?, 2), dtype=float32)
Tensor("decoder_state_in_hidden_process/Elu:0", shape=(?, 13), dtype=float32)
Tensor("decoder_state_in_process/Elu:0", shape=(?, 14), dtype=float32)
Tensor("hidden_decoder_rnn_layer/rnn/transpose:0", shape=(53, 682, 14), dtype=float32)
Tensor("hidden_decoder_rnn_layer/rnn/while/Exit_2:0", shape=(?, 14), dtype=float32)
Tensor("decoder_rnn_layer/rnn/transpose:0", shape=(53, 682, 14), dtype=float32)
Tensor("decoder_rnn_layer/rnn/while/Exit_2:0", shape=(?, 14), dtype=float32)
Tensor("decoder_outs/Reshape:0", shape=(36146, 14), dtype=float32)
Tensor("readout_affine/Identity:0", shape=(36146, 1), dtype=float32)
Tensor("readout_affine/Reshape:0", shape=(53, 682), dtype=float32)
Tensor("error/mul_1:0", shape=(53, 682), dtype=float32)
Tensor("error/Mean:0", shape=(), dtype=float32)
Tensor("error/Mean:0", shape=(), dtype=float32)
Can you write Python code for it?


I am working on the file "04_time_series_prediction/arima/arima_price_history.ipynb".
The first task is:
Train - Test
Can you write Python code for it?

XX = series.values
XX.shape

#splitting
train_size = int(len(XX)*2/3)
train_size

train_set = XX[:train_size]
test_set = XX[train_size:]
train_set.shape, test_set.shape

train_set[0]

history = list(train_set)

history[0] = 3333

history = list(train_set)
len(history)

predictions = []

for tt in range(len(test_set)):
    output = ARIMA(history, order=(5,1,0)).fit(disp=0).forecast()
    y_hat = output[0] #0th is the index of the prediction
    predictions.append(y_hat)

    observation = test_set[tt]
    #history.append(observation)
    history.append(y_hat)

    print "predicted: {}, expected: {}".format(y_hat, observation)

error = mean_squared_error(predictions, test_set)
error

plt.figure(figsize=(15,6))
plt.plot(predictions, label='preds')
plt.plot(test_set, label='test set')
plt.legend()
plt.show()


I am working on the file "04_time_series_prediction/arima/arima_price_history.ipynb".
The first task is:
http://machinelearningmastery.com/arima-for-time-series-forecasting-with-python/
http://ucanalytics.com/blogs/arima-models-manufacturing-case-study-example-part-3/
https://stats.stackexchange.com/questions/23036/estimating-same-model-over-multiple-time-series
The idea with ARIMA models is that the final residual should look like white noise otherwise, there is juice or information available in the data to extract
Can you write Python code for it?

from __future__ import division
from os import path, remove
import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import StratifiedShuffleSplit
from time import time
from matplotlib import pyplot as plt
import seaborn as sns
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
import shutil
from tensorflow.contrib.learn.python.learn import learn_runner
from sklearn.metrics import r2_score
from fastdtw import fastdtw
from collections import OrderedDict
from scipy.spatial.distance import euclidean
from statsmodels.tsa.stattools import coint
from skopt.space.space import Integer, Real
from skopt import gp_minimize
from skopt.plots import plot_convergence
import pickle
import inspect
import dill
import sys
from pandas import read_csv
from pandas import datetime
from pandas.plotting import autocorrelation_plot
from statsmodels.graphics.tsaplots import plot_pacf
from statsmodels.tsa.arima_model import ARIMA
from pandas import DataFrame
from sklearn.metrics import mean_squared_error

%matplotlib inline

data_path = '../../../../../Dropbox/data'

score_dic_filepath = data_path + "/arima/scoredic_testing.npy"

test = np.load(score_dic_filepath)[()]
len(test.keys())

ph_data_path = data_path + '/price_history'

npz_path = ph_data_path + '/price_history_mobattrs_date_dp_60to30_62020_trimmed'

npz_train = npz_path + '_train.npz'
assert path.isfile(npz_train)
path.abspath(npz_train)

dic = np.load(npz_train)

dic.keys()

dic = {
    "inputs": dic['inputs'][:, :, 0],
    "targets": dic['targets'],
    #"full": np.hstack((dic['inputs'][:, :, 0], dic['targets'])),
}

for key, val in dic.iteritems():
    print key, val.shape

#fix targets
fulls = np.array([np.concatenate( (item_in, item_tar + item_in[-1]) )
 for item_in, item_tar in zip(dic['inputs'], dic['targets'])])

fulls.shape

sns.tsplot(fulls[100])

autocorrelation_plot(fulls[100])

plot_pacf(fulls[100])
plt.show()


I am working on the file "04_training_linear_models.ipynb".
The first task is:
And let's set the random seed so the output of this exercise solution is reproducible:
Can you write Python code for it?


I am working on the file "04_training_linear_models.ipynb".
The first task is:
And now let's measure the final model's accuracy on the test set:
Can you write Python code for it?

logits = X_test.dot(Theta)
Y_proba = softmax(logits)
y_predict = np.argmax(Y_proba, axis=1)

accuracy_score = np.mean(y_predict == y_test)
accuracy_score