OculusQuestで『Puppo, The Corgi』を動かす

# はじめに
[『Puppo, The Corgi』― Unity ML-Agents Toolkit を活用した可愛さ溢れるデモゲーム](https://blogs.unity3d.com/jp/2018/10/02/puppo-the-corgi-cuteness-overload-with-the-unity-ml-agents-toolkit/)をOculusQuestで動かしてみました。
PC上で動かすのであれば上記ページからダウンロードしたそのままで動かせるのですが、ML-Agents 0.5.0で作成されているため、OculusQuestで動かすためにAndroid向けにビルドするとエラーになってしまいます。
Android向けにビルドできるように最新版のML-Agentsで動かせるように修正したので、修正した内容について書いておきます。
<blockquote class="twitter-tweet"><p lang="ja" dir="ltr">Puppoかわいい<a href="https://t.co/ap0JexR3l0">https://t.co/ap0JexR3l0</a> <a href="https://t.co/xaankte6ni">pic.twitter.com/xaankte6ni</a></p>&mdash; 高浜 (@SatoshiTakahama) <a href="https://twitter.com/SatoshiTakahama/status/1257806874180636673?ref_src=twsrc%5Etfw">May 5, 2020</a></blockquote> <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>
#開発環境
Unity 2019.3.12f1
ML-Agents 1.0.0
Barracuda 0.7.0

#修正したところ
・UnityのPackage ManagerからBarracuda 0.7.0をインストールします。
・ML-Agentsの1.0.0を[ダウンロード](https://github.com/Unity-Technologies/ml-agents/releases)し、チュ－トリアルのAssets/ML-Agentsフォルダを差し替えます。
・[インストールガイド](https://github.com/Unity-Technologies/ml-agents/blob/master/docs/Installation.md)に従い、com.unity.ml-agentsUnityパッケージをインストールします。
・DogAgent.csを以下のように書き換えます。

```C#:DogAgent.cs
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Unity.MLAgents;
using Unity.MLAgents.Sensors;
using Unity.MLAgentsExamples;

public class DogAgent : Agent {

    [HideInInspector]
    // The target the dog will run towards.
    public Transform target; 

    // These items should be set in the inspector
    [Header("Body Parts")] 
    public Transform mouthPosition;
    public Transform body;
    public Transform leg0_upper;
    public Transform leg1_upper;
    public Transform leg2_upper;
    public Transform leg3_upper;
    public Transform leg0_lower;
    public Transform leg1_lower;
    public Transform leg2_lower;
    public Transform leg3_lower;

    // These determine how the dog should be able to rotate around the y axis
    [Header("Body Rotation")] 
    public float maxTurnSpeed;
    public ForceMode turningForceMode;

    [Header("Sounds")]
    // If true, the dog will bark. 
    // Note : This should be turned off during training...unless you want to hear a dozen dogs barking for hours
    public bool canBark;
    // The clips to use for the barks of the dog
    public List<AudioClip> barkSounds = new List <AudioClip>();
	AudioSource audioSourceSFX;

    JointDriveController jdController;
    
    [HideInInspector]
    // This vector gives the position of the target relative to the position of the dog
	public Vector3 dirToTarget;
    // This float determines how much the dog will be rotating around the y axis
    float rotateBodyActionValue; 
    // Counts the number of steps until the next agent's decision will be made
    int decisionCounter;

    // [HideInInspector]
    public bool runningToItem;
    // [HideInInspector]
    public bool returningItem;

    void Awake()
    {
        // Audio Setup
        audioSourceSFX = body.gameObject.AddComponent<AudioSource>();
        audioSourceSFX.spatialBlend = .75f;
        audioSourceSFX.minDistance = .7f;
        audioSourceSFX.maxDistance = 5;
        if(canBark)
        {
            StartCoroutine(BarkBarkGame());
        }

        //Joint Drive Setup
        jdController = GetComponent<JointDriveController>();
        jdController.SetupBodyPart(body);
        jdController.SetupBodyPart(leg0_upper);
        jdController.SetupBodyPart(leg0_lower);
        jdController.SetupBodyPart(leg1_upper);
        jdController.SetupBodyPart(leg1_lower);
        jdController.SetupBodyPart(leg2_upper);
        jdController.SetupBodyPart(leg2_lower);
        jdController.SetupBodyPart(leg3_upper);
        jdController.SetupBodyPart(leg3_lower);

    }

    /// <summary>
    /// Add relevant information on each body part to observations.
    /// </summary>
    public void CollectObservationBodyPart(BodyPart bp, VectorSensor sensor)
    {
        var rb = bp.rb;
        sensor.AddObservation(bp.groundContact.touchingGround ? 1 : 0); // Is this bp touching the ground
        if(bp.rb.transform != body)
        {
            sensor.AddObservation(bp.currentXNormalizedRot);
            sensor.AddObservation(bp.currentYNormalizedRot);
            sensor.AddObservation(bp.currentZNormalizedRot);
            sensor.AddObservation(bp.currentStrength/jdController.maxJointForceLimit);
        }
    }

    /// <summary>
    /// The method the agent uses to collect information about the environment
    /// </summary>
    public override void CollectObservations(VectorSensor sensor)
    {
        sensor.AddObservation(dirToTarget.normalized);
        sensor.AddObservation(body.localPosition);
        sensor.AddObservation(jdController.bodyPartsDict[body].rb.velocity);
        sensor.AddObservation(jdController.bodyPartsDict[body].rb.angularVelocity);
        sensor.AddObservation(body.forward); //the capsule is rotated so this is local forward
        sensor.AddObservation(body.up); //the capsule is rotated so this is local forward
        foreach (var bodyPart in jdController.bodyPartsDict.Values)
        {
            CollectObservationBodyPart(bodyPart, sensor);
        }
    }


    /// <summary>
    /// Rotates the body of the agent around the y axis
    /// </summary>
    /// <param name="act"> The amount by which the agent must rotate</param>
    void RotateBody(float act)
    {
        float speed = Mathf.Lerp(0, maxTurnSpeed, Mathf.Clamp(act, 0, 1));
        Vector3 rotDir = dirToTarget; 
        rotDir.y = 0;
        // Adds a force on the front of the body
        jdController.bodyPartsDict[body].rb.AddForceAtPosition(
            rotDir.normalized * speed * Time.deltaTime, body.forward, turningForceMode); 
        // Adds a force on the back od the body
        jdController.bodyPartsDict[body].rb.AddForceAtPosition(
            -rotDir.normalized * speed * Time.deltaTime, -body.forward, turningForceMode); 
    }

    /// <summary>
    /// Allows the dog to bark
    /// </summary>
    /// <returns></returns>
    public IEnumerator BarkBarkGame()
    {   

        while(true)
        {
            //When we're returning the stick we should not bark because we have
            //a stick in our mouth :|>
            if(!returningItem)
            {
                //Choose one of the barking clips at random and play it.
                audioSourceSFX.PlayOneShot(barkSounds[Random.Range( 0, barkSounds.Count)], 1);
            }
            //Wait for a random amount of time (between 1 & 10 sec) until we bark again.
            yield return new WaitForSeconds(Random.Range(1, 10)); 
        }
    }

    /// <summary>
    /// The agent's action method. Is called at each decision and allows the agent to move
    /// </summary>
    /// <param name="vectorAction"> The actions that were determined by the policy</param>
    /// <param name="textAction"> The text action given by the policy</param>
	public override void OnActionReceived(float[] vectorAction)
    {

        var bpDict = jdController.bodyPartsDict;
      
        // Update joint drive target rotation
        bpDict[leg0_upper].SetJointTargetRotation(vectorAction[0], vectorAction[1], 0);
        bpDict[leg1_upper].SetJointTargetRotation(vectorAction[2], vectorAction[3], 0);
        bpDict[leg2_upper].SetJointTargetRotation(vectorAction[4], vectorAction[5], 0);
        bpDict[leg3_upper].SetJointTargetRotation(vectorAction[6], vectorAction[7], 0);
        bpDict[leg0_lower].SetJointTargetRotation(vectorAction[8], 0, 0);
        bpDict[leg1_lower].SetJointTargetRotation(vectorAction[9], 0, 0);
        bpDict[leg2_lower].SetJointTargetRotation(vectorAction[10], 0, 0);
        bpDict[leg3_lower].SetJointTargetRotation(vectorAction[11], 0, 0);

        // Update joint drive strength
        bpDict[leg0_upper].SetJointStrength(vectorAction[12]);
        bpDict[leg1_upper].SetJointStrength(vectorAction[13]);
        bpDict[leg2_upper].SetJointStrength(vectorAction[14]);
        bpDict[leg3_upper].SetJointStrength(vectorAction[15]);
        bpDict[leg0_lower].SetJointStrength(vectorAction[16]);
        bpDict[leg1_lower].SetJointStrength(vectorAction[17]);
        bpDict[leg2_lower].SetJointStrength(vectorAction[18]);
        bpDict[leg3_lower].SetJointStrength(vectorAction[19]);

        rotateBodyActionValue = vectorAction[20];
  
    }

    /// <summary>
    /// Update the direction vector to the current target;
    /// </summary>
    public void UpdateDirToTarget()
    {
        dirToTarget = target.position - jdController.bodyPartsDict[body].rb.position;

    }

    void FixedUpdate()
    {
        UpdateDirToTarget();

        if (decisionCounter == 0)
        {
            decisionCounter = 3;
            RequestDecision();
        }
        else
        {
            decisionCounter--;
        }

        RotateBody(rotateBodyActionValue); 

        // Energy Conservation
        // The dog is penalized by how strongly it rotates towards the target.
        // Without this penalty the dog tries to rotate as fast as it can at all times.
        var bodyRotationPenalty = -0.001f * rotateBodyActionValue;
        AddReward(bodyRotationPenalty);

        // Reward for moving towards the target
        RewardFunctionMovingTowards();
        // Penalty for time
        RewardFunctionTimePenalty();
    }
	
    /// <summary>
    /// Reward moving towards target & Penalize moving away from target.
    /// This reward incentivizes the dog to run as fast as it can towards the target,
    /// and decentivizes running away from the target.
    /// </summary>
    void RewardFunctionMovingTowards()
    {

		float movingTowardsDot = Vector3.Dot(
		    jdController.bodyPartsDict[body].rb.velocity, dirToTarget.normalized); 
        AddReward(0.01f * movingTowardsDot);
    }


    /// <summary>
    /// Time penalty
    /// The dog gets a pentalty each step so that it tries to finish 
    /// as quickly as possible.
    /// </summary>
    void RewardFunctionTimePenalty()
    {
        AddReward(- 0.001f);  //-0.001f chosen by experimentation.
    }

}
```
・DogAcademy.csを以下のように書き換えます。

```C#:DogAcademy.cs
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Unity.MLAgents;
using Unity.MLAgentsExamples;

public class DogAcademy : MonoBehaviour
{

    public void AcademyReset()
    {
        Monitor.verticalOffset = 1f;
        Physics.defaultSolverIterations = 12;
        Physics.defaultSolverVelocityIterations = 12;
        Time.captureFramerate = 0;
    }

    public void AcademyStep()
    {
    }

}

```
・エディタ拡張のコード(Assets/PuppoTheCorgi/Fetch/Editorフォルダのファイル)についてはなくても動かせるので、ビルドを通すためにとりあえず削除かコメントアウトします。
・CORGIオブジェクトに、Behavior Parametersスクリプト、DecisionRequesteスクリプトを付加します。付加した後は以下のようになります。
![image.png](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/215278/1eb57b8e-8667-76a0-47bc-b4ae30ff0330.png)
・トレーニング用のパラメータは、以下のようにマニュアルに記載されていた通りで動作しました。

```
CorgiLearning:
    normalize: true
    num_epoch: 3
    time_horizon: 1000
    batch_size: 2048
    buffer_size: 20480
    gamma: 0.995
    max_steps: 2e6
    summary_freq: 3000
    num_layers: 3
    hidden_units: 512
```
・トレーニング開始は以下コマンドです。

```
mlagents-learn config/trainer_config.yaml --run-id corgi_01
```
#追記
・棒を手でつかんで投げるために、STICKオブジェクトにXRGrabInteractableスクリプトをつけて、手を放したときにThrow.csのThrowItemメソッドを呼ぶようにしました。
![image.png](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/215278/c65bce16-d929-9403-a51e-b99440a56343.png)
・Throw.csは以下のように書き換えました。

```C#:Throw.cs
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class Throw : MonoBehaviour {

	[Header("GENERAL")]
	// When true, the player can throw the stick
	public bool canThrow; 
	// A reference to the item the player is allowed to throw
	public Transform item;
	// A reference to the stick in the dog's mouth when the game starts
	public GameObject stickTitleScreen;

	// The position of the player. This is where the dog will return the stick
	public Transform returnPoint; 

	[HideInInspector]
	// The rigidbody of item
	public Rigidbody itemRB; 
	[HideInInspector]
	// Collider of item
	public Collider itemCol; 
	[HideInInspector]
	// The dog the player is playing with
	public DogAgent dogAgent; 

	[Header("HOLDING ITEM SETTINGS")]
	// The offset postion relative to the player
	public Vector3 holdingPositionOffset; 
	// The position of the stick relative to the player when holding
	Vector3 holdingPos; 
	// Velocity to use when holding the stick
	public float holdingItemTargetVelocity; 
	// Max velocity change allowed for the stick when being held
	public float holdingItemMaxVelocityChange; 

	[Header("THROWING PARAMETERS")]
	// Defines the strength of the player
	public float throwSpeed; 
	// Direction the player is thowing towards
	private Vector3 throwDir; 

	[Header("SOUND")]
	// List of throwing sounds clips
	public List<AudioClip> throwSounds = new List <AudioClip>();

	//SFX audio source
	AudioSource audioSourceSFX; 
	// Starting position of touch/mouse click
	Vector3 startingPos; 
	// Current position of touch/mouse click
	Vector3 currentPos; 
	// Is true when the player is currently touching/clicking the screen
	bool currentlyTouching; 
	// Current Touch detected
	Touch currentTouch; 
	// Is true when touch input detected
	bool usingTouchInput; 
	// Is true when mouse input detected
	bool usingMouseInput;
	// Camera in the scene
	Camera cam; 

	/// <summary>
	/// Initialization iof the Throw component
	/// </summary>
	void Awake ()
	{
		cam = Camera.main;
		dogAgent = FindObjectOfType<DogAgent>();
		itemRB = item.GetComponent<Rigidbody>();
		itemCol = item.GetComponent<Collider>();
		audioSourceSFX = gameObject.AddComponent<AudioSource>();
		dogAgent.target = returnPoint;
	}
	
	/// <summary>
	/// Is called when the player swipes the screen
	/// </summary>
	void StartSwipe()
	{
		startingPos = cam.ScreenToViewportPoint(Input.mousePosition) - new Vector3(0.5f, 0.5f, 0.0f);
		usingTouchInput = true;
		currentlyTouching = true;
		currentTouch = Input.GetTouch(0);
		if(!dogAgent.returningItem)
		{
			dogAgent.target = item;
		}
	}

	/// <summary>
	/// Is called when the player drags the mouse on the screen
	/// </summary>
	void StartMouseDrag()
	{
		startingPos = cam.ScreenToViewportPoint(Input.mousePosition) - new Vector3(0.5f, 0.5f, 0.0f);
		usingMouseInput = true;
		currentlyTouching = true;
		if(!dogAgent.returningItem)
		{
			dogAgent.target = item;
		}
	}

	/// <summary>
	/// This method is called when the player is throwing the item
	/// </summary>
	public void ThrowItem()
	{
		canThrow = false;
		audioSourceSFX.PlayOneShot(throwSounds[Random.Range( 0, throwSounds.Count)], .25f);
#if UNITY_EDITOR
        itemRB.velocity *= .5f;
		throwDir = (currentPos - startingPos);
		var dir = cam.transform.TransformDirection(throwDir) + cam.transform.forward;
		dir.y = 0;
		itemRB.AddForce(dir * throwSpeed, ForceMode.VelocityChange);
#endif
        StartCoroutine(DelayedThrow());
	}

	/// <summary>
	/// This Coroutine ensures the dog will wait a moment before going toward the target
	/// </summary>
	/// <returns></returns>
	IEnumerator DelayedThrow()
	{
		float elapsed = 0;
		while(elapsed < 2)
		{
			elapsed += Time.deltaTime;
			yield return null;
		}
		StartCoroutine(GoGetItemGame());
	}

	void FixedUpdate()
	{
#if UNITY_EDITOR
        if (currentlyTouching)
		{
			if(usingTouchInput)
			{
				currentTouch = Input.GetTouch(0);

				currentPos = cam.ScreenToViewportPoint(currentTouch.position) - new Vector3(0.5f, 0.5f, 0.0f);
			}
			if(usingMouseInput)
			{
				currentPos = cam.ScreenToViewportPoint(Input.mousePosition) - new Vector3(0.5f, 0.5f, 0.0f);
			}
			holdingPos = cam.transform.TransformPoint(holdingPositionOffset + (currentPos * 2));
			Vector3 moveToPos = holdingPos - itemRB.position;  //cube needs to go to the standard Pos
			Vector3 velocityTarget = moveToPos * holdingItemTargetVelocity * Time.deltaTime; //not sure of the logic here, but it modifies velTarget
            itemRB.velocity = Vector3.MoveTowards(itemRB.velocity, velocityTarget, holdingItemMaxVelocityChange);
		}
#endif
    }

    void Update()
	{
#if UNITY_EDITOR
		if(canThrow)
		{
			if (Input.touchCount > 0 && !currentlyTouching)
			{
				currentTouch = Input.GetTouch(0);
				if(currentTouch.phase == TouchPhase.Began)
				{
					StartSwipe();
				}
			}

			if(usingTouchInput && currentlyTouching)
			{
				currentTouch = Input.GetTouch(0);
				if(currentTouch.phase == TouchPhase.Ended)
				{
					currentlyTouching = false;
					ThrowItem();
				}
			}

			if (Input.GetMouseButtonDown(0) && !currentlyTouching)
			{
				StartMouseDrag();
			}

			if(usingMouseInput && currentlyTouching)
			{
				if(Input.GetMouseButtonUp(0))
				{
					currentlyTouching = false;
					ThrowItem();
				}
			}
        }
#endif
    }
	
	/// <summary>
	/// The dog just picked the item up. We set it's target to be the player
	/// </summary>
	public void PickUpItemGame()
	{
		//Make the stick kinematic and put it in the dog's mouth
		itemCol.enabled = false; //Disable the collider on the stick.
		itemRB.isKinematic = true; //Turn off physics
		item.position = dogAgent.mouthPosition.position; //Set stick position
		item.rotation = dogAgent.mouthPosition.rotation; //Set stick rotation
		item.SetParent(dogAgent.mouthPosition); //Parent the stick to the dog's mouth
		dogAgent.runningToItem = false; //We are no longer running towards the stick

		//The stick is now in the dog's mouth so we need to change
		//the dog's target to the return point
		dogAgent.target = returnPoint; //set the dog's target to the return point
		dogAgent.UpdateDirToTarget(); //update the direction vector
		dogAgent.returningItem = true; //we are not returning the stick
	}

	/// <summary>
	/// The dog just droped the stick at the position of the player
	/// </summary>
	public void DropItemGame()
	{
		itemRB.isKinematic = false; //Enable physics on the stick 
		item.parent = null; //Stick no longer parented to the dog
		itemCol.enabled = true; //Re-enable the collider on the stick
		dogAgent.returningItem = false; //We are done returning the stick
		canThrow = true; //Dog has dropped the stick. We can now throw it again
	}
	
	/// <summary>
	/// Triggered when the player throws the item.
	/// </summary>
	/// <returns></returns>
	 public IEnumerator GoGetItemGame()
    {   
        Debug.Log("STARTING GoGetItemGame()");

        //GO GET THE STICK
        dogAgent.target = item; //Set the target to the stick.
        dogAgent.runningToItem = true; //We are now running towards the stick
        yield return new WaitForSeconds(0.1f);

        //Wait till we are in range of the stick
        while (dogAgent.dirToTarget.sqrMagnitude > 1f) //wait until we are close
        {
            yield return null;
        }
		//Since we are in proximity to the stick
		//We should pick up the stick
        PickUpItemGame();
        yield return new WaitForSeconds(0.1f);

        //Wait till we are in range of the return point
        while (dogAgent.dirToTarget.sqrMagnitude > 1f) //wait until we are close
        {
            yield return null;
        }

		//Since we are back at the return point
		//We should drop the stick
        DropItemGame();

        Debug.Log("ENDING GoGetItemGame()");
    }

}

```