<!DOCTYPE html>
<html>
<head>
    <meta charset="utf-8" />
    <title>Metacar: Documentation</title>
    <!-- Load the latest version of TensorFlow.js -->
    <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.11.6"> </script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/pixi.js/4.7.1/pixi.min.js"></script>
</head>
<body>

    <div id="env"></div>

    <script src="https://cdn.jsdelivr.net/combine/npm/metacar@0.1.1,npm/metacar@0.1.1"></script>
    <script>

        // Select a level
        const level = metacar.level.level0;
        // Create the environement
        const env = new metacar.env("env", level);
        // Set the agent motion
        env.setAgentMotion(metacar.motion.BasicMotion, {rotationStep: 0.1});

        // Create the model
        // Input
        const input = tf.input({batchShape: [null, 26]});
        // Hidden layer
        const layer = tf.layers.dense({useBias: true, units: 32, activation: 'relu'}).apply(input);
        // Output layer
        const output = tf.layers.dense({useBias: true, units: 3, activation: 'linear'}).apply(layer);
        // Create the model
        const model = tf.model({inputs: input, outputs: output});
        // Optimize
        let model_optimizer = tf.train.adam(0.01);

        // Loss of the model
        function model_loss(tf_states, tf_actions, Qtargets){
            return tf.tidy(() => {
                // valeur
                return model.predict(tf_states).sub(Qtargets).square().mul(tf_actions).mean();
            });
        }

        // Pick an action eps-greedy
        function pickAction(st, eps){
            let st_tensor = tf.tensor([st]);
            let act;
            if (Math.random() < eps){ // Pick a random action
                act = Math.floor(Math.random()*3);
            }
            else {
                let result = model.predict(st_tensor);
                let argmax = result.argMax(1);
                act = argmax.buffer().values[0];
                argmax.dispose();
                result.dispose();
            }
            st_tensor.dispose();
            return act;
        }

        // Return the mean of an array
        function mean(array){
            if (array.length == 0)
                return null;
            var sum = array.reduce(function(a, b) { return a + b; });
            var avg = sum / array.length;
            return avg;
        }

        // Train the model
        function train_model(states, actions, rewards, next_states){
            var size = next_states.length;
            // Transform each array into a tensor
            let tf_states = tf.tensor2d(states, shape=[states.length, 26]);
            let tf_rewards = tf.tensor2d(rewards, shape=[rewards.length, 1]);
            let tf_next_states = tf.tensor2d(next_states, shape=[next_states.length, 26]);
            let tf_actions = tf.tensor2d(actions, shape=[actions.length, 3]);
            // Get the list of loss to compute the mean later in this function
            let losses = []

            // Get the QTargets
            const Qtargets = tf.tidy(() => {
                let Q_stp1 = model.predict(tf_next_states);
                let Qtargets = tf.tensor2d(Q_stp1.max(1).expandDims(1).mul(tf.scalar(0.99)).add(tf_rewards).buffer().values, shape=[size, 1]);
                return Qtargets;
            });

            // Generate batch of training and train the model
            let batch_size = 32;
            for (var b = 0; b < size; b+=32) {

                // Select the batch
                let to = (b + batch_size < size) ?  batch_size  : (size - b);
                const tf_states_b = tf_states.slice(b, to);
                const tf_actions_b = tf_actions.slice(b, to);
                const Qtargets_b = Qtargets.slice(b, to);

                // Minimize the error
                model_optimizer.minimize(() => {
                    const loss = model_loss(tf_states_b, tf_actions_b, Qtargets_b);
                    losses.push(loss.buffer().values[0]);
                    return loss;
                });

                // Dispose the tensors from the memory
                tf_states_b.dispose();
                tf_actions_b.dispose();
                Qtargets_b.dispose();
            }

            console.log("Mean loss", mean(losses));

            // Dispose the tensors from the memory
            Qtargets.dispose();
            tf_states.dispose();
            tf_rewards.dispose();
            tf_next_states.dispose();
            tf_actions.dispose();
        }

        env.load().then(() => {

            env.addEvent("play", () => {
                // Move forward
                let st = env.getState().linear;
                let act = pickAction(st, 0.0);
                let reward = env.step(act);
                console.log(reward);
                // Log the reward
            });

            env.addEvent("stop", () => {
                return;
            });

            env.addEvent("train", async () => {

                let eps = 1.0;
                // Used to store the experiences
                let states = [];
                let rewards = [];
                let reward_mean = [];
                let next_states = [];
                let actions = [];

                // Get the current state of the lidar
                let st = env.getState().linear;
                let st2;

                for (let epi=0; epi < 150; epi++){
                    let reward = 0;
                    let step = 0;
                    while (step < 400){
                        // pick an action
                        let act = pickAction(st, eps);
                        reward = env.step(act);
                        st2 = env.getState().linear;


                        let mask = [0, 0, 0];
                        mask[act] = 1;

                        // Randomly insert the new transition tuple
                        let index = Math.floor(Math.random() * states.length);
                        states.splice(index, 0, st);
                        rewards.splice(index, 0, [reward]);
                        reward_mean.splice(index, 0, reward)
                        next_states.splice(index, 0, st2);
                        actions.splice(index, 0, mask);
                        // Be sure to keep the size of the dataset under 10000 transitions
                        if (states.length > 10000){
                            states = states.slice(1, states.length);
                            rewards = rewards.slice(1, rewards.length);
                            reward_mean = reward_mean.slice(1, reward_mean.length);
                            next_states = next_states.slice(1, next_states.length);
                            actions = actions.slice(1, actions.length);
                        }

                        st = st2;
                        step += 1;
                    }
                    // Decrease epsilon
                    eps = Math.max(0.1, eps*0.99);

                    // Train model every 5 episodes
                    if (epi % 5 == 0){
                        console.log("---------------");
                        console.log("rewards mean", mean(reward_mean));
                        console.log("episode", epi);
                        await train_model(states, actions, rewards, next_states);
                        await tf.nextFrame();
                    }
                    // Shuffle the env
                    env.shuffle();
                }

                env.render(true);
            });

        });

    </script>

</body>
</html>

Transform ce programe en python