AudioSourceの音声データをUnity上でVisualizeする

# 背景

AudioのVisualize力を上げたいなと思って、UnityのAudioSource周りのAPIを調べてみた。
いくつかサンプルを書いてみたので、Audio+Unityで遊ぶ人の参考になれば幸い.

# 成果物

サンプルを作ってみた.
[動画](https://www.youtube.com/watch?v=au3M9MWdez8)

<iframe width="560" height="315" src="https://www.youtube.com/embed/au3M9MWdez8" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>

1. 波形をそのまま表示
1. 周波数成分をそのまま表示
1. 音の大きさに合わせてtransform変更 (scale,rotation,position)

上記3つをやってみた

# 基本

AudioSourceで知っとくと良さそうな基本のAPIをいくつか

```csharp
AudioSource source;

// timeSamples: intで現在の実行中のサンプリング位置を取得する.　例: 44,100Hzでサンプリング時、0.5sec後には `22,050`が返ってくる.
source.timeSamples;

// GetData: AudioClipのデータを `float[]` で取得する.
var data = new float[source.clip.channels * source.clip.samples];
source.clip.GetData(data, 0);

// GetSpectrumData: FFTされたデータを取得する. offset, 窓関数も指定できる
var spectram = new float[2048];
source.GetSpectrumData(spectram, 0, FFTWindow.BlackmanHarris);

// frequency: サンプリング周波数. 例: 44,100Hz
source.clip.frequency;
```


# 1. 波形をそのまま表示

AudioClipからGetDataして、それを一定時間ごとにLineRendererを使って表示することで波形の表示できる.

```csharp
public class SequenceLineRenderer : MonoBehaviour, IMusicRender
{
    [SerializeField] private LineRenderer lineRenderer;
    [SerializeField] private float waveLength = 20.0f;
    [SerializeField] private float yLength = 10f;

    private AudioSource source = default;
    private float[] data = default;
    private int sampleStep = default;
    private Vector3[] samplingLinePoints = default;

    private void Start()
    {
        var source = GetComponent<AudioSource>();
        var clip = source.clip;
        var data = new float[clip.channels * clip.samples];
        source.clip.GetData(data, 0);

        Prepare(source, data);
    }

    public void Prepare(AudioSource source, float[] data)
    {
        this.source = source;
        this.data = data;

        var fps = Mathf.Max(60f, 1f / Time.fixedDeltaTime);
        var clip = source.clip;
        sampleStep = (int) (clip.frequency / fps);
        samplingLinePoints = new Vector3[sampleStep];
    }

    private void FixedUpdate()
    {
        if (source.isPlaying && source.timeSamples < data.Length)
        {
            var startIndex = source.timeSamples;
            var endIndex = source.timeSamples + sampleStep;
            Inflate(
                data, startIndex, endIndex,
                samplingLinePoints,
                waveLength, -waveLength / 2f, yLength
            );
            Render(samplingLinePoints);
        }
        else
        {
            Reset();
        }
    }

    private void Render(Vector3[] points)
    {
        if (points == null) return;
        lineRenderer.positionCount = points.Length;
        lineRenderer.SetPositions(points);
    }

    private void Reset()
    {
        var x = -waveLength / 2;
        Render(new[]
        {
            new Vector3(-x, 0, 0) + this.transform.position,
            this.transform.position,
            new Vector3(x, 0, 0) + this.transform.position,
        });
    }

    public void Inflate(
        float[] target, int start, int end,
        Vector3[] result,
        float xLength, float xOffset, float yLength
    )
    {
        var samples = Math.Max(end - start, 1f);
        var xStep = xLength / samples;
        var j = 0;

        for (var i = start; i < end; i++, j++)
        {
            var x = xOffset + xStep * j;
            var y = i < target.Length ? target[i] * yLength : 0f;
            var p = new Vector3(x, y, 0) + this.transform.position;
            result[j] = p;
        }
    }
}
```

# 2. 周波数成分をそのまま表示する

同じノリでfftしたデータをLineRendererで表示する.

```csharp
public class SpectramLineRenderer : MonoBehaviour, IMusicRender
{
    [SerializeField] private LineRenderer lineRenderer;
    [SerializeField] private float waveLength = 20.0f;
    [SerializeField] private float yLength = 10f;

    private AudioSource source = null;
    private float[] spectram = null;
    private Vector3[] points = null;
    private const int FFT_RESOLUTION = 2048;

    private void Start()
    {
        var source = GetComponent<AudioSource>();
        var clip = source.clip;
        var data = new float[clip.channels * clip.samples];
        source.clip.GetData(data, 0);

        Prepare(source, data);
    }

    public void Prepare(AudioSource source, float[] data)
    {
        this.source = source;
        this.spectram = new float[FFT_RESOLUTION];
        this.points = new Vector3[FFT_RESOLUTION];
    }

    public void FixedUpdate()
    {
        Render();
    }

    private void Render()
    {
        source.GetSpectrumData(spectram, 0, FFTWindow.BlackmanHarris);
        var xStart = -waveLength / 2;
        var xStep = waveLength / spectram.Length;
        for (var i = 0; i < points.Length; i++)
        {
            var y = spectram[i] * yLength;
            var x = xStart + xStep * i;
            var p = new Vector3(x, y, 0) + transform.position;
            points[i] = p;
        }

        Render(points);
    }

    private void Render(Vector3[] points)
    {
        if (points == null) return;
        lineRenderer.positionCount = points.Length;
        lineRenderer.SetPositions(points);
    }

    private void Reset()
    {
        var x = waveLength / 2;
        Render(new[]
        {
            new Vector3(-x, 0, 0) + transform.position,
            new Vector3(0, 0, 0) + transform.position,
            new Vector3(x, 0, 0) + transform.position,
        });
    }
}
```

# 3. 音量に合わせてscaleを変化させる

区間内の最大の振れ幅を取得して、表示する.
そのままだと変化が大きすぎるのでlerpしてあげるとみやすい

```csharp
public class VolumeSizeRenderer : MonoBehaviour
{
    [SerializeField] private float scaleFactor = 1f;
    [SerializeField] private float lerp = 0.5f;

    private AudioSource source = default;
    private float[] data = default;
    private Vector3 initialScale = default;
    private int sampleStep = default;

    private void Awake()
    {
        this.initialScale = transform.localScale;
    }

    private void Start()
    {
        var source = GetComponent<AudioSource>();
        var clip = source.clip;
        var data = new float[clip.channels * clip.samples];
        source.clip.GetData(data, 0);

        Prepare(source, data);
    }

    public void Prepare(AudioSource source, float[] monoData)
    {
        this.source = source;
        this.data = monoData;

        var fps = Mathf.Max(60f, 1f / Time.fixedDeltaTime);
        var clip = source.clip;
        this.sampleStep = (int) (clip.frequency / fps);
    }

    private void FixedUpdate()
    {
        if (source.isPlaying && source.timeSamples < data.Length)
        {
            var startIndex = source.timeSamples;
            var endIndex = Math.Min(source.timeSamples + sampleStep, data.Length);
            var level = DetectVolumeLevel(data, startIndex, endIndex);
            Render(level);
        }
        else
        {
            Reset();
        }
    }

    private void Render(float size)
    {
        var diff = initialScale * this.scaleFactor * size;
        transform.localScale = Vector3.Lerp(transform.localScale, diff, lerp);
    }

    private void Reset()
    {
        transform.localScale = initialScale;
    }

    private float DetectVolumeLevel(float[] data, int start, int end)
    {
        var max = 0f;
        var min = 0f;

        for (var i = start; i < end; i++)
        {
            if (max < data[i]) max = data[i];
            if (min > data[i]) min = data[i];
        }

        return max - min;
    }
}
```

# まとめ

とくにライブラリとかもなく、Unityのデフォルトの機能でFFTまでデータ取得できるのは便利..!
色々応用が効くので、trasnform意外もいじってみたい