[Note] ML-Agent with Unity-chan

## Introduction
Unity provides brilliant framework to apply RL to the game object.
In this article, I would like Unity-chan to learn walking to the target object using Tensorflow.

## Procedure
1. Finish the basic "Get started project" of [**3DBall**](https://github.com/Unity-Technologies/ml-agents/blob/master/docs/Basic-Guide.md)
2. Finish [this tutorial about how to create your own game from scratch](https://github.com/Unity-Technologies/ml-agents/blob/master/docs/Learning-Environment-Create-New.md)
3. Get [**Unity-chan**](https://assetstore.unity.com/packages/3d/characters/unity-chan-model-18705)
<img width="333" alt="Screen Shot 2018-06-25 at 0.38.15.png" src="https://qiita-image-store.s3.amazonaws.com/0/240801/cfdb5a93-52e3-a84d-3de8-b7c3f1aa07e5.png">

4. Place Unity-chan onto the plane.
<img width="1089" alt="Screen Shot 2018-06-25 at 0.41.49.png" src="https://qiita-image-store.s3.amazonaws.com/0/240801/a47e630e-99ca-c848-16f3-09ccdaf1029d.png">

5. Create animator controller.
<img width="692" alt="Screen Shot 2018-06-25 at 0.42.30.png" src="https://qiita-image-store.s3.amazonaws.com/0/240801/97ee9aec-4b1d-5de7-e6ac-266658882ddc.png">

6. Set the parameter "IsRunning" true to the transition from "idle" to "run".
<img width="1255" alt="Screen Shot 2018-06-25 at 0.43.22.png" src="https://qiita-image-store.s3.amazonaws.com/0/240801/d5cf4899-7953-2f31-b9cf-6c13e06fb98c.png">
7. Set the parameter "IsRunning" false to the transition from "idle" to "run".
8. <img width="1251" alt="Screen Shot 2018-06-25 at 0.44.11.png" src="https://qiita-image-store.s3.amazonaws.com/0/240801/c98cbaf3-9cfd-edb4-30da-e2ffb8066efc.png">
9. Set this animator controller to Unity-chan.
10. Create Script name "Controller" in C#.

```c#:Controller.cs
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using MLAgents;

public class Controller : Agent {

    Rigidbody rb;

    public float speed = 2f;
    public float thrust = 100;
    private Animator animator;

    bool ground;

	// Use this for initialization
	void Start () {
        rb = GetComponent<Rigidbody>();
        animator = GetComponent<Animator>();
	}

    public void goRight(){
        rb.velocity = new Vector3(speed, 0, 0);
        transform.rotation = Quaternion.Euler(0, 90, 0);
        animator.SetBool("IsRunning", true);
    }

    public void goLeft()
    {
        rb.velocity = new Vector3(-speed, 0, 0);
        transform.rotation = Quaternion.Euler(0, 270, 0);
        animator.SetBool("IsRunning", true);
    }

    public void goUp()
    {
        rb.velocity = new Vector3(0, 0, speed);
        transform.rotation = Quaternion.Euler(0, 0, 0);
        animator.SetBool("IsRunning", true);
    }

    public void goDown()
    {
        rb.velocity = new Vector3(0, 0, -speed);
        transform.rotation = Quaternion.Euler(0, 180, 0);
        animator.SetBool("IsRunning", true);
    }

	private void OnCollisionStay(Collision collision)
	{
        ground = true;
	}

    public void DecisionMaker(int data){
        Debug.Log(data);

        if(data == 1){
            goRight();
        } else if(data == 2){
            goLeft();
        } else if(data == 3){
            goUp();
        } else {
            goDown();
        }
    }

    public Transform Target;
    public override void AgentReset()
    {
        if (this.transform.position.y < -1.0)
        {
            this.transform.position = Vector3.zero;
            this.rb.angularVelocity = Vector3.zero;
            this.rb.velocity = Vector3.zero;
        }
        else
        {
            Target.position = new Vector3(Random.value * 8 - 4,
                                          0.5f,
                                          Random.value * 8 - 4);
        }
    }

    public override void CollectObservations()
    {
        Vector3 relativePosition = Target.position - this.transform.position;

        AddVectorObs(relativePosition.x / 5);
        AddVectorObs(relativePosition.z / 5);

        AddVectorObs((this.transform.position.x + 5) / 5);
        AddVectorObs((this.transform.position.x - 5) / 5);
        AddVectorObs((this.transform.position.z + 5) / 5);
        AddVectorObs((this.transform.position.z - 5) / 5);

        AddVectorObs(rb.velocity.x / 5);
        AddVectorObs(rb.velocity.z / 5);
    }

    private float previousDistance = float.MaxValue;

    public override void AgentAction(float[] vectorAction, string textAction)
    {
        float distanceToTarget = Vector3.Distance(this.transform.position, Target.position);

        if (distanceToTarget < 1.42f)
        {
            AddReward(1.0f);
            Done();
        }

        if (distanceToTarget < previousDistance)
        {
            AddReward(0.1f);
        }

        AddReward(-0.05f);

        if (this.transform.position.y < -1.0)
        {
            AddReward(-1.0f);
            Done();
        }

        previousDistance = distanceToTarget;

        // vectorAction contains a discrete value.
        DecisionMaker(System.Convert.ToInt32(vectorAction.GetValue(0)));
    }
}
```
11. Then follow the same direction as we did in [the tutorial](https://github.com/Unity-Technologies/ml-agents/blob/master/docs/Basic-Guide.md)

**Command**
`python3 learn.py --run-id=<run-identifier> --train`

## Demo
![Untitled.gif](https://qiita-image-store.s3.amazonaws.com/0/240801/3bd6ab57-5f2f-3ff4-00a1-2a9fd5c92273.gif)

## License
© UTJ/UCL