Repository: PaddlePaddle/models

Issue:
Add a tutorial for policy gradient
[A rendered version tutorial](https://wanghaoshuang.github.io/2017/12/Policy-Gradient-RL-by-PaddlePaddle/)
 内容为:
 - 任务描述
 -  模型
 -  策略（目标函数）
 -  算法（Gradient ascent）
 -  PaddlePaddle实现



Discussion:

Resolving patch:
diff --git a/fluid/policy_gradient/brain.py b/fluid/policy_gradient/brain.py
new file mode 100644
--- /dev/null
+++ b/fluid/policy_gradient/brain.py
@@ -0,0 +1,99 @@
+import numpy as np
+import paddle.v2 as paddle
+import paddle.v2.fluid as fluid
+# reproducible
+np.random.seed(1)
+
+
+class PolicyGradient:
+    def __init__(
+            self,
+            n_actions,
+            n_features,
+            learning_rate=0.01,
+            reward_decay=0.95,
+            output_graph=False, ):
+        self.n_actions = n_actions
+        self.n_features = n_features
+        self.lr = learning_rate
+        self.gamma = reward_decay
+
+        self.ep_obs, self.ep_as, self.ep_rs = [], [], []
+
+        self.place = fluid.CPUPlace()
+        self.exe = fluid.Executor(self.place)
+
+    def build_net(self):
+
+        obs = fluid.layers.data(
+            name='obs', shape=[self.n_features], dtype='float32')
+        acts = fluid.layers.data(name='acts', shape=[1], dtype='int64')
+        vt = fluid.layers.data(name='vt', shape=[1], dtype='float32')
+        # fc1
+        fc1 = fluid.layers.fc(
+            input=obs,
+            size=10,
+            act="tanh"  # tanh activation
+        )
+        # fc2
+        self.all_act_prob = fluid.layers.fc(input=fc1,
+                                            size=self.n_actions,
+                                            act="softmax")
+        # to maximize total reward (log_p * R) is to minimize -(log_p * R)
+        neg_log_prob = fluid.layers.cross_entropy(
+            input=self.all_act_prob,
+            label=acts)  # this is negative log of chosen action
+        neg_log_prob_weight = fluid.layers.elementwise_mul(x=neg_log_prob, y=vt)
+        loss = fluid.layers.reduce_mean(
+            x=neg_log_prob_weight)  # reward guided loss
+
+        sgd_optimizer = fluid.optimizer.SGD(self.lr)
+        sgd_optimizer.minimize(loss)
+        self.exe.run(fluid.default_startup_program())
+
+    def choose_action(self, observation):
+        prob_weights = self.exe.run(
+            fluid.default_main_program().prune(self.all_act_prob),
+            feed={"obs": observation[np.newaxis, :]},
+            fetch_list=[self.all_act_prob])
+        prob_weights = np.array(prob_weights[0])
+        action = np.random.choice(
+            range(prob_weights.shape[1]),
+            p=prob_weights.ravel())  # select action w.r.t the actions prob
+        return action
+
+    def store_transition(self, s, a, r):
+        self.ep_obs.append(s)
+        self.ep_as.append(a)
+        self.ep_rs.append(r)
+
+    def learn(self):
+        # discount and normalize episode reward
+        discounted_ep_rs_norm = self._discount_and_norm_rewards()
+        tensor_obs = np.vstack(self.ep_obs).astype("float32")
+        tensor_as = np.array(self.ep_as).astype("int64")
+        tensor_as = tensor_as.reshape([tensor_as.shape[0], 1])
+        tensor_vt = discounted_ep_rs_norm.astype("float32")[:, np.newaxis]
+        # train on episode
+        self.exe.run(
+            fluid.default_main_program(),
+            feed={
+                "obs": tensor_obs,  # shape=[None, n_obs]
+                "acts": tensor_as,  # shape=[None, ]
+                "vt": tensor_vt  # shape=[None, ]
+            })
+        self.ep_obs, self.ep_as, self.ep_rs = [], [], []  # empty episode data
+        return discounted_ep_rs_norm
+
+    def _discount_and_norm_rewards(self):
+        # discount episode rewards
+        discounted_ep_rs = np.zeros_like(self.ep_rs)
+        running_add = 0
+        for t in reversed(range(0, len(self.ep_rs))):
+            running_add = running_add * self.gamma + self.ep_rs[t]
+            discounted_ep_rs[t] = running_add
+
+        # normalize episode rewards
+        discounted_ep_rs -= np.mean(discounted_ep_rs)
+        discounted_ep_rs /= np.std(discounted_ep_rs)
+        return discounted_ep_rs
diff --git a/fluid/policy_gradient/env.py b/fluid/policy_gradient/env.py
new file mode 100644
--- /dev/null
+++ b/fluid/policy_gradient/env.py
@@ -0,0 +1,56 @@
+import time
+import sys
+import numpy as np
+
+
+class Env():
+    def __init__(self, stage_len, interval):
+        self.stage_len = stage_len
+        self.end = self.stage_len - 1
+        self.position = 0
+        self.interval = interval
+        self.step = 0
+        self.epoch = -1
+        self.render = False
+
+    def reset(self):
+        self.end = self.stage_len - 1
+        self.position = 0
+        self.epoch += 1
+        self.step = 0
+        if self.render:
+            self.draw(True)
+
+    def status(self):
+        s = np.zeros([self.stage_len]).astype("float32")
+        s[self.position] = 1
+        return s
+
+    def move(self, action):
+        self.step += 1
+        reward = 0.0
+        done = False
+        if action == 0:
+            self.position = max(0, self.position - 1)
+        else:
+            self.position = min(self.end, self.position + 1)
+        if self.render:
+            self.draw()
+        if self.position == self.end:
+            reward = 1.0
+            done = True
+        return reward, done, self.status()
+
+    def draw(self, new_line=False):
+        if new_line:
+            print ""
+        else:
+            print "\r",
+        for i in range(self.stage_len):
+            if i == self.position:
+                sys.stdout.write("O")
+            else:
+                sys.stdout.write("-")
+        sys.stdout.write("    epoch: %d; steps: %d" % (self.epoch, self.step))
+        sys.stdout.flush()
+        time.sleep(self.interval)
diff --git a/fluid/policy_gradient/run.py b/fluid/policy_gradient/run.py
new file mode 100644
--- /dev/null
+++ b/fluid/policy_gradient/run.py
@@ -0,0 +1,29 @@
+from brain import PolicyGradient
+from env import Env
+import numpy as np
+
+n_actions = 2
+interval = 0.01
+stage_len = 10
+epoches = 10000
+
+if __name__ == "__main__":
+
+    brain = PolicyGradient(n_actions, stage_len)
+    e = Env(stage_len, interval)
+    brain.build_net()
+    done = False
+
+    for epoch in range(epoches):
+        if (epoch % 500 == 1) or epoch < 5 or epoch > 3000:
+            e.render = True
+        else:
+            e.render = False
+        e.reset()
+        while not done:
+            s = e.status()
+            action = brain.choose_action(s)
+            r, done, _ = e.move(action)
+            brain.store_transition(s, action, r)
+        done = False
+        brain.learn()


Test patch: