jsdoでtensorflow.js その17

#概要

jsdoでtensorflow.jsやってみた。
さらに、dqn見つけたので、やってみた。

#写真

ランダム
![](http://jsdo-static-contents.s3.amazonaws.com/images/capture/i/5/V/i5VJ.jpg?t=1541501373)

dqn
![image](https://qiita-image-store.s3.amazonaws.com/0/18104/15590b5c-ae00-f430-1176-46edd08ea0d7.png)

#サンプルコード

```
var Cartpole = function() {
    this.canvas = document.getElementById('canvas');
    this.ctx = this.canvas.getContext('2d');
	this.gravity = 9.8;
    this.masscart = 1.0;
	this.masspole = 0.1;
	this.total_mass = (this.masspole + this.masscart);
	this.length = 0.5;
	this.polemass_length = (this.masspole * self.length);
	this.force_mag = 10.0;
	this.tau = 0.02;
	this.theta_threshold_radians = 12 * 2 * Math.PI / 360;
	this.x_threshold = 2.4;
    this.state = {
       x: 0,
       x_dot: 0,
       theta: 0,
       theta_dot: 0
    };
	this.steps_beyond_done = null;
};
Cartpole.prototype.reset = function() {
	this.state = {
       x: Math.random() * 0.4 - 0.2,
       x_dot: Math.random() * 0.4 - 0.2,
       theta: Math.random() * 0.4 - 0.2,
       theta_dot: Math.random() * 0.4 - 0.2
    };
	this.steps_beyond_done = null;
	return this.state;
}
Cartpole.prototype.step = function(action) {
	var state = this.state;
	var x = state.x; 
    var x_dot = state.x_dot;
    var theta = state.theta;
    var theta_dot = state.theta_dot;
	var force = this.force_mag;
    if (action < 1) force = -this.force_mag;
	var costheta = Math.cos(theta);
	var sintheta = Math.sin(theta);
	var temp = (force + this.polemass_length * theta_dot * theta_dot * sintheta) / this.total_mass;
	var thetaacc = (this.gravity * sintheta - costheta * temp) / (this.length * (4.0 / 3.0 - this.masspole * costheta * costheta / this.total_mass));
	var xacc = temp - this.polemass_length * thetaacc * costheta / this.total_mass;
	x = x + this.tau * x_dot
	x_dot = x_dot + this.tau * xacc;
	theta = theta + this.tau * theta_dot;
	theta_dot = theta_dot + this.tau * thetaacc;
    this.state = {
        x: x, 
        x_dot: x_dot, 
        theta: theta, 
        theta_dot: theta_dot
    };
    var reward;
	var done = x < -this.x_threshold || x > this.x_threshold || theta < -this.theta_threshold_radians || theta > this.theta_threshold_radians;
    if (!done) 
    {
        reward = 1.0;
    }
    else if (this.steps_beyond_done == null)
    {
	    this.steps_beyond_done = 0;
		reward = 1.0;
    }
    else
    {
        if (this.steps_beyond_done == 0) 
        {
            alert("err");
        }
		this.steps_beyond_done += 1;
		reward = 0.0;
    }
    return [this.state, reward, done];
}
Cartpole.prototype.render = function() {
    var scale = 100;
    var x = this.state.x;
    var theta = this.state.theta;
	var cartx = x * scale + this.canvas.width / 2.0;
	this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
    this.ctx.beginPath();
    this.ctx.strokeStyle = '#ffa500';
    this.ctx.lineWidth = 8;
    this.ctx.moveTo(cartx - 20, 200);
    this.ctx.lineTo(cartx + 20, 200);
    this.ctx.stroke();
    this.ctx.beginPath();
    this.ctx.strokeStyle = 'black';
    this.ctx.lineWidth = 2;
    this.ctx.moveTo(cartx, 200);
    this.ctx.lineTo(cartx + Math.sin(theta) * 100, 200 - Math.cos(theta) * 100);
    this.ctx.stroke();
    return 0;
}
Cartpole.prototype.getStateTensor = function() {
    return tf.tensor2d([[this.state.x, this.state.x_dot, this.state.theta, this.state.theta_dot]]);
}
var env = new Cartpole();
class PolicyNetwork {
    constructor(hiddenLayerSizesOrModel) {
        if (hiddenLayerSizesOrModel instanceof tf.Model) 
        {
            this.model = hiddenLayerSizesOrModel;
        }
        else
        {
            this.createModel(hiddenLayerSizesOrModel);
        }
    }
    createModel(hiddenLayerSizes) {
        if (!Array.isArray(hiddenLayerSizes)) 
        {
            hiddenLayerSizes = [hiddenLayerSizes];
        }
        this.model = tf.sequential();
        hiddenLayerSizes.forEach((hiddenLayerSize, i) => {
            this.model.add(tf.layers.dense({
                units: hiddenLayerSize,
                activation: 'elu',
                inputShape: i === 0 ? [4] : undefined
            }));
        });
        this.model.add(tf.layers.dense({
            units: 1
        }));
    }
    train(env, optimizer, discountRate, numGames, maxStepsPerGame) {
        const allGradients = [];
        const allRewards = [];
        const gameSteps = [];
        for (let i = 0; i < numGames; ++i) 
        {
            const gameRewards = [];
            const gameGradients = [];
            var observation = env.reset();
            for (let j = 0; j < maxStepsPerGame; ++j)
            {

                env.render();         
                const gradients = tf.tidy(() => {
                    const inputTensor = env.getStateTensor();
                    return this.getGradientsAndSaveActions(inputTensor).grads;
                });
                this.pushGradients(gameGradients, gradients);
                const action = this.currentActions_[0];
                var res = env.step(action);
                var observation = res[0];
                var reward = res[1];
                var isDone = res[2];            
                if (isDone) 
                {
                    gameRewards.push(0);
                    break;
                }
                else 
                {
                    gameRewards.push(1);
                }
            }
            gameSteps.push(gameRewards.length);
            this.pushGradients(allGradients, gameGradients);
            allRewards.push(gameRewards);
            tf.nextFrame();
        }
        tf.tidy(() => {
            const normalizedRewards = discountAndNormalizeRewards(allRewards, discountRate);
            optimizer.applyGradients(scaleAndAverageGradients(allGradients, normalizedRewards));
        });
        tf.dispose(allGradients);
        return gameSteps;
    }
    getGradientsAndSaveActions(inputTensor) {
        const f = () => tf.tidy(() => {
            const [logits, actions] = this.getLogitsAndActions(inputTensor);
            this.currentActions_ = actions.dataSync();
            const labels = tf.sub(1, tf.tensor2d(this.currentActions_, actions.shape));
            return tf.losses.sigmoidCrossEntropy(labels, logits).asScalar();
        });
        return tf.variableGrads(f);
    }
    getCurrentActions() {
        return this.currentActions_;
    }
    getLogitsAndActions(inputs) {
        return tf.tidy(() => {
            const logits = this.model.predict(inputs);
            const leftProb = tf.sigmoid(logits);
            const leftRightProbs = tf.concat([leftProb, tf.sub(1, leftProb)], 1);
            const actions = tf.multinomial(leftRightProbs, 1, null, true);
            return [logits, actions];
        });
    }
    getActions(inputs) {
        return this.getLogitsAndActions(inputs)[1].dataSync();
    }
    pushGradients(record, gradients) {
        for (const key in gradients) 
        {
            if (key in record)
            {
                record[key].push(gradients[key]);
            }
            else
            {
                record[key] = [gradients[key]];
            }
        }
    }
}
const MODEL_SAVE_PATH_ = 'indexeddb://cart-pole-v1';
class SaveablePolicyNetwork extends PolicyNetwork {
    constructor(hiddenLayerSizesOrModel) {
        super(hiddenLayerSizesOrModel);
    }
    saveModel() {
        return this.model.save(MODEL_SAVE_PATH_);
    }
    loadModel() {
        const modelsInfo = tf.io.listModels();
        if (MODEL_SAVE_PATH_ in modelsInfo)
        {
            const model = tf.loadModel(MODEL_SAVE_PATH_);
            return new SaveablePolicyNetwork(model);
        }
        else
        {
            throw new Error(`Cannot find model at ${MODEL_SAVE_PATH_}.`);
        }
    }
    checkStoredModelStatus() {
        const modelsInfo = tf.io.listModels();
        return modelsInfo[MODEL_SAVE_PATH_];
    }
    removeModel() {
        return tf.io.removeModel(MODEL_SAVE_PATH_);
    }
    hiddenLayerSizes() {
        const sizes = [];
        for (let i = 0; i < this.model.layers.length - 1; ++i)
        {
            sizes.push(this.model.layers[i].units);
        }
        return sizes.length === 1 ? sizes[0] : sizes;
    }
}
function discountRewards(rewards, discountRate) {
    const discountedBuffer = tf.buffer([rewards.length]);
    let prev = 0;
    for (let i = rewards.length - 1; i >= 0; --i)
    {
        const current = discountRate * prev + rewards[i];
        discountedBuffer.set(current, i);
        prev = current;
    }
    return discountedBuffer.toTensor();
}
function discountAndNormalizeRewards(rewardSequences, discountRate) {
    return tf.tidy(() => {
        const discounted = [];
        for (const sequence of rewardSequences) 
        {
            discounted.push(discountRewards(sequence, discountRate))
        }
        const concatenated = tf.concat(discounted);
        const mean = tf.mean(concatenated);
        const std = tf.sqrt(tf.mean(tf.square(concatenated.sub(mean))));
        const normalized = discounted.map(rs => rs.sub(mean).div(std));
        return normalized;
    });
}
function scaleAndAverageGradients(allGradients, normalizedRewards) {
    return tf.tidy(() => {
        const gradients = {};
        for (const varName in allGradients)
        {
            gradients[varName] = tf.tidy(() => {
                const varGradients = allGradients[varName].map(varGameGradients => tf.stack(varGameGradients));
                const expandedDims = [];
                for (let i = 0; i < varGradients[0].rank - 1; ++i)
                {
                    expandedDims.push(1);
                }
                const reshapedNormalizedRewards = normalizedRewards.map(rs => rs.reshape(rs.shape.concat(expandedDims)));
                for (let g = 0; g < varGradients.length; ++g)
                {
                    varGradients[g] = varGradients[g].mul(reshapedNormalizedRewards[g]);
                }
                return tf.mean(tf.concat(varGradients, 0), 0);
            });
        }
        return gradients;
    });
}
const optimizer = tf.train.adam(0.01);
var policyNet = new PolicyNetwork(20);
var steps = policyNet.train(env, optimizer, 0.9, 100, 200);
var canvas = document.getElementById('canvas');
var ctx = this.canvas.getContext('2d');
ctx.beginPath();
ctx.strokeStyle = 'black';
ctx.lineWidth = 1;
ctx.moveTo(0, 300);
for (var i = 0; i < 100; i++)
{
    ctx.lineTo(i * 10, 300 - steps[i] * 1.5);
    
}
ctx.stroke();



```

#成果物

http://jsdo.it/ohisama1/o2Od

以上。