Update .gitattributes

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+yolov8n.sentis filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+*.meta

README.md

@@ -0,0 +1,42 @@
+---
+pipeline_tag: object-detection
+---
+
+
+# YOLOv8n validated for Unity Sentis (Version 1.4.0-pre.3*)
+
+This is Yolov8n object detection example code for unity sentis, not real-time inference, for instance image.
+
+** this is almost same code with ref, but only for image. **
+
+ref: https://huggingface.co/unity/sentis-YOLOv8n
+
+[YOLOv8n](https://docs.ultralytics.com/models/yolov8/) is a real-time multi-object recognition model confirmed to run in Unity 2023.
+
+## How to Use
+First get the package `com.unity.sentis` from the package manager.
+You will also need the Unity UI package.
+
+* Create a new scene in Unity 2023 or higher (include in Unity 6000).
+* Install `com.unity.sentis` version `1.4.0-pre.3` from the package manager
+* Add the c# script to the Main Camera.
+* Create a Raw Image in the scene and link it as the `displayImage`
+* Drag the yolov8n.sentis or yolov8n.onnx file into the model asset field
+* Drag the classes.txt on to the labelAssets field
+* Place a *.png image file in the Assets folder and drag onto Input Texture field
+* Set the fields for the bounding box texture sprite (you can [create your own one](https://docs.unity3d.com/Manual/9SliceSprites.html) using a transparent texture or use an inbuilt one) and the font
+
+
+## Preview
+If working correctly you should see something like this:
+
+![preview](preview.png)
+
+## Information
+The NMS selection will be improved in later versions of Sentis. Currently uses singular-class approach.
+
+## Unity Sentis
+Unity Sentis is the inference engine that runs in Unity 3D. More information can be found at [here](https://unity.com/products/sentis)
+
+## License
+The YOLO models use the GPLv3 license.

classes.txt

@@ -0,0 +1,80 @@
+person
+bicycle
+car
+motorbike
+aeroplane
+bus
+train
+truck
+boat
+traffic light
+fire hydrant
+stop sign
+parking meter
+bench
+bird
+cat
+dog
+horse
+sheep
+cow
+elephant
+bear
+zebra
+giraffe
+backpack
+umbrella
+handbag
+tie
+suitcase
+frisbee
+skis
+snowboard
+sports ball
+kite
+baseball bat
+baseball glove
+skateboard
+surfboard
+tennis racket
+bottle
+wine glass
+cup
+fork
+knife
+spoon
+bowl
+banana
+apple
+sandwich
+orange
+broccoli
+carrot
+hot dog
+pizza
+donut
+cake
+chair
+sofa
+potted plant
+bed
+dining table
+toilet
+tv monitor
+laptop
+mouse
+remote
+keyboard
+cell phone
+microwave
+oven
+toaster
+sink
+refrigerator
+book
+clock
+vase
+scissors
+teddy bear
+hair drier
+toothbrush

yolo.cs

@@ -0,0 +1,238 @@
+using System.Collections.Generic;
+using Unity.Sentis;
+using UnityEngine;
+using UnityEngine.UI;
+using UnityEngine.Video;
+using FF = Unity.Sentis.Functional;
+
+/*
+ *  YOLOv8n Inference Script
+ *  ========================
+ *
+ * This is YOLOv8n's Instance image version.
+ * Almost same code, but image. ref: https://huggingface.co/unity/sentis-YOLOv8n
+ * 
+ * Place this script on the Main Camera.
+ * 
+ * Place the yolob8n.sentis file in the asset folder and drag onto the asset field
+ * Place a *.png image file in the Assets folder and drag onto Input Texture field
+ * Create a RawImage in your scene and set it as the displayImage field
+ * Drag the classes.txt into the labelsAsset field
+ * Set Border Sprite to UIMask
+ * Add a reference to a sprite image for the bounding box and a font for the text
+ * 
+ */
+public class yolo : MonoBehaviour
+{
+    public ModelAsset modelAsset;
+    public TextAsset labelsAsset;
+    public RawImage displayImage;
+    public Sprite borderSprite;
+    public Texture2D borderTexture;
+    public Texture2D inputTexture;
+    public Font font;
+
+    private Transform displayLocation;
+    private string[] labels;
+    private Model model;
+    private IWorker engine;
+    private RenderTexture targetRT;
+    private TensorFloat inputTensor;
+    private const BackendType backend = BackendType.GPUCompute;
+    
+    // Image Size for model
+    private const int imageWidth = 640;
+    private const int imageHeight = 640;
+
+    List<GameObject> boxPool = new List<GameObject>();
+    TensorFloat centersToCorners;
+    
+    [SerializeField, Range(0, 1)] float iouThreshold = 0.5f;
+    [SerializeField, Range(0, 1)] float scoreThreshold = 0.5f;
+    public struct BoundingBox
+    {
+        public float centerX;
+        public float centerY;
+        public float width;
+        public float height;
+        public string label;
+    }
+    
+    void Start()
+    {
+        labels = labelsAsset.text.Split('\n');
+        
+        // Load Model
+        LoadModel();
+        
+        displayLocation = displayImage.transform;
+
+        SetupInput();
+        
+        if (borderSprite == null)
+        {
+            borderSprite = Sprite.Create(borderTexture, new Rect(0, 0, borderTexture.width, borderTexture.height), new Vector2(borderTexture.width / 2, borderTexture.height / 2));
+        }
+        
+        ExecuteML();
+    }
+
+    // Update is called once per frame
+    
+    void LoadModel()
+    {
+
+        // Load model
+        var model1 = ModelLoader.Load(modelAsset);
+
+        centersToCorners = new TensorFloat(new TensorShape(4, 4),
+            new float[]
+            {
+                1,      0,      1,      0,
+                0,      1,      0,      1,
+                -0.5f,  0,      0.5f,   0,
+                0,      -0.5f,  0,      0.5f
+            });
+
+        //Here we transform the output of the model1 by feeding it through a Non-Max-Suppression layer.
+        var model2 = Functional.Compile(
+            input =>
+            {
+                var modelOutput = model1.Forward(input)[0];
+                var boxCoords = modelOutput[0, 0..4, ..].Transpose(0, 1);        //shape=(8400,4)
+                var allScores = modelOutput[0, 4.., ..];                         //shape=(80,8400)
+                var scores = FF.ReduceMax(allScores, 0) - scoreThreshold;        //shape=(8400)
+                var classIDs = FF.ArgMax(allScores, 0);                          //shape=(8400) 
+                var boxCorners = FF.MatMul(boxCoords, FunctionalTensor.FromTensor(centersToCorners));
+                var indices = FF.NMS(boxCorners, scores, iouThreshold);           //shape=(N)
+                var indices2 = indices.Unsqueeze(-1).BroadcastTo(new int[] { 4 });//shape=(N,4)
+                var coords = FF.Gather(boxCoords, 0, indices2);                  //shape=(N,4)
+                var labelIDs = FF.Gather(classIDs, 0, indices);                  //shape=(N)
+                return (coords, labelIDs);
+            },
+            InputDef.FromModel(model1)[0]
+        );
+
+        //Create engine to run model
+        engine = WorkerFactory.CreateWorker(backend, model2);
+    }
+
+
+    void SetupInput()
+    {
+        inputTensor = TextureConverter.ToTensor(inputTexture, imageWidth, imageHeight);
+    }
+
+    public void ExecuteML()
+    {
+        displayImage.texture = inputTexture;
+
+        engine.Execute(inputTensor);
+        
+        var output = engine.PeekOutput("output_0") as TensorFloat;
+        var labelIDs = engine.PeekOutput("output_1") as TensorInt;
+        
+        output.CompleteOperationsAndDownload();
+        labelIDs.CompleteOperationsAndDownload();
+
+        float displayWidth = displayImage.rectTransform.rect.width;
+        float displayHeight = displayImage.rectTransform.rect.height;
+
+        float scaleX = displayWidth / imageWidth;
+        float scaleY = displayHeight / imageHeight;
+        
+        int foundBoxes = output.shape[0];
+        
+        //Draw the bounding boxes
+        for (int n = 0; n < foundBoxes; n++)
+        {
+            var box = new BoundingBox
+            {
+                centerX = output[n, 0] * scaleX - displayWidth / 2,
+                centerY = output[n, 1] * scaleY - displayHeight / 2,
+                width = output[n, 2] * scaleX,
+                height = output[n, 3] * scaleY,
+                label = labels[labelIDs[n]],
+            };
+            
+            // It's for Debug
+            Debug.Log(box.label);
+            
+            DrawBox(box, n, displayHeight * 0.05f);
+        }
+    }
+    public void DrawBox(BoundingBox box, int id, float fontSize)
+    {
+        //Create the bounding box graphic or get from pool
+        GameObject panel;
+        if (id < boxPool.Count)
+        {
+            panel = boxPool[id];
+            panel.SetActive(true);
+        }
+        else
+        {
+            panel = CreateNewBox(Color.yellow);
+        }
+        //Set box position
+        panel.transform.localPosition = new Vector3(box.centerX, -box.centerY);
+
+        //Set box size
+        RectTransform rt = panel.GetComponent<RectTransform>();
+        rt.sizeDelta = new Vector2(box.width, box.height);
+
+        //Set label text
+        var label = panel.GetComponentInChildren<Text>();
+        label.text = box.label;
+        label.fontSize = (int)fontSize;
+    }
+
+    public GameObject CreateNewBox(Color color)
+    {
+        //Create the box and set image
+
+        var panel = new GameObject("ObjectBox");
+        panel.AddComponent<CanvasRenderer>();
+        Image img = panel.AddComponent<Image>();
+        img.color = color;
+        img.sprite = borderSprite;
+        img.type = Image.Type.Sliced;
+        panel.transform.SetParent(displayLocation, false);
+
+        //Create the label
+
+        var text = new GameObject("ObjectLabel");
+        text.AddComponent<CanvasRenderer>();
+        text.transform.SetParent(panel.transform, false);
+        Text txt = text.AddComponent<Text>();
+        txt.font = font;
+        txt.color = color;
+        txt.fontSize = 40;
+        txt.horizontalOverflow = HorizontalWrapMode.Overflow;
+
+        RectTransform rt2 = text.GetComponent<RectTransform>();
+        rt2.offsetMin = new Vector2(20, rt2.offsetMin.y);
+        rt2.offsetMax = new Vector2(0, rt2.offsetMax.y);
+        rt2.offsetMin = new Vector2(rt2.offsetMin.x, 0);
+        rt2.offsetMax = new Vector2(rt2.offsetMax.x, 30);
+        rt2.anchorMin = new Vector2(0, 0);
+        rt2.anchorMax = new Vector2(1, 1);
+
+        boxPool.Add(panel);
+        return panel;
+    }
+    
+    public void ClearAnnotations() 
+    {
+        foreach (var box in boxPool)
+        {
+            box.SetActive(false);
+        }
+    }
+
+    private void OnDestroy()
+    {
+        centersToCorners?.Dispose();
+        engine?.Dispose();
+    }
+}

yolov8n.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15504f8b73630dd71768e776474262ba0d9fc472820601b1ac1528bdb6cfec95
+size 12823637

yolov8n.sentis

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6d067ead11259acbf633be97f7780ac1c1b68ef7a1851959c79254d940de9f3
+size 12834372