[Swift]骨格検出アプリ作成備忘録　2.動画に対して骨格検出を行う　

# 骨格検出アプリ作成２
前回、動画の呼び出しまでできたのでその読み込んだ動画に対して骨格検出を行いたいと思います。

https://qiita.com/jinzin5535/items/74fa2a662d092f4c2935

# できたもの
![kokkaku_demo.gif](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/3571351/4efd7db7-823c-900a-417c-a6a77a165707.gif)


# 仕組み
1.読み込んだ動画(a)をフレーム単位で静止画(b)にする
2.(b)に対して骨格検出を行い、骨格が表示された静止画(c)を作成する
3.(c)をドキュメント内に再生順に保存していく
4.(c)群を並べて動画にする
5.動画にする際の再生レートを(a)を参照して設定する

色々と方法を考えてはみましたが、知識不足と相まって回りくどいやり方になってしまいました。ここに至るまで、至ってからも色々と大変ポイントがあったのですが、とりあえず割愛します。

# コード1.読み込んだ動画(a)をフレーム単位で静止画(b)にする

```swift
private func trimVideo(){
        let trimAsset = (videoPlayer?.currentItem?.asset)!
        let composition = AVMutableComposition()
        guard let videoTrack = composition.addMutableTrack(withMediaType: .video, preferredTrackID: kCMPersistentTrackID_Invalid) else {
            debugPrint("Failed to add video track")
            return
        }
        // 素材Assetの1個目のVideoトラックを使う
        let srcVideoTrack = trimAsset.tracks(withMediaType: .video)[0]
        do {
            // CMTimeMakeSecondsはpreferredTimescaleを100とすることで秒数が正確に表せる
            try videoTrack.insertTimeRange(CMTimeRangeMake(start: CMTimeMakeWithSeconds(cutStartTime, preferredTimescale: 100), duration: CMTimeMakeWithSeconds(cutEndTime,preferredTimescale: 100)), of: srcVideoTrack, at: .zero)
        } catch { print(error) }
        self.cutplayerItem = AVPlayerItem(asset: composition)
        self.cutvideo = AVPlayer(playerItem: self.cutplayerItem)
        
        // 動画をフレームに直す処理準備
        let cutAsset = cutvideo?.currentItem?.asset
        let assetReader = try! AVAssetReader(asset: cutAsset!)
        let CutvideoTrack = cutAsset!.tracks(withMediaType: .video)[0]
        let outputSettings: [String: Any] = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
        let trackOutput = AVAssetReaderTrackOutput(track: CutvideoTrack, outputSettings: outputSettings)
        assetReader.add(trackOutput)
        
        // 読み込みの開始
        assetReader.startReading()
        var frameNumber = 0
        
        while let sampleBuffer = trackOutput.copyNextSampleBuffer(){
            guard let pixcelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
                continue
            }
            if CMSampleBufferGetImageBuffer(sampleBuffer) != nil {
                let ciImage = CIImage(cvPixelBuffer: pixcelBuffer)
                let context = CIContext(options: nil)
                let cgImage = context.createCGImage(ciImage, from: ciImage.extent)!
                let image = UIImage(cgImage: cgImage)
                // 骨格検出関数　image:取り出したフレーム,frameNumber:ファイル名保存用の変数
                self.poseDetect(image: image, frameNumber: frameNumber)
                frameNumber += 1
            } else {
                break
            }
        }
    }
```

# コード2.(b)に対して骨格検出を行い、骨格が表示された静止画(c)を作成する
```swift
// 人物の関節点と関節点同士を結んだ骨格線を予測する
    func poseDetect(image:UIImage, frameNumber:Int){
        
        // 画像に直接合成して保存する場合は画像のサイズを参照する
        let windowSize_w = image.size.width
        let windowSize_h = image.size.height
        // 処理したい画像（フレーム）を保持する関数
        let imageRequestHandler = VNImageRequestHandler(cgImage:image.cgImage!)
        // 画像中の人物の骨格情報を推定するもの（モデル含む？）
        let poseEstimationRequest = VNDetectHumanBodyPoseRequest()
        
        do {
            // poseEstimationRequestを使ってimageRequestHandlerの画像を解析する
            try imageRequestHandler.perform([poseEstimationRequest])
        } catch {
            print("error")
        }
        guard let results = poseEstimationRequest.results as? [VNHumanBodyPoseObservation] else{return}
        do {
            for obserbation in results {
                // 右腕
                let rightShoulder = try obserbation.recognizedPoint(.rightShoulder).location
                let rightElbow = try obserbation.recognizedPoint(.rightElbow).location
                let rightWrist = try obserbation.recognizedPoint(.rightWrist).location
                
                //  左腕
                let leftShoulder = try obserbation.recognizedPoint(.leftShoulder).location
                let leftElbow = try obserbation.recognizedPoint(.leftElbow).location
                let leftWrist = try obserbation.recognizedPoint(.leftWrist).location
                
                // 腰
                let root = try obserbation.recognizedPoint(.root).location
                
                // 右脚
                let rightHip = try obserbation.recognizedPoint(.rightHip).location
                let rightKnee = try obserbation.recognizedPoint(.rightKnee).location
                let rightAnkle = try obserbation.recognizedPoint(.rightAnkle).location
                
                //  左脚
                let leftHip = try obserbation.recognizedPoint(.leftHip).location
                let leftKnee = try obserbation.recognizedPoint(.leftKnee).location
                let leftAnkle = try obserbation.recognizedPoint(.leftAnkle).location
                
                let RS_x = rightShoulder.x * windowSize_w
                let RS_y = (1 - rightShoulder.y) * windowSize_h
                let RS_cir = UIBezierPath(arcCenter: CGPoint(x: RS_x, y: RS_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let RS_lay = CAShapeLayer()
                RS_lay.path = RS_cir.cgPath
                RS_lay.fillColor = UIColor.red.cgColor
                RS_lay.lineWidth = 3.0
                
                let RE_x = rightElbow.x * windowSize_w
                let RE_y = (1 - rightElbow.y) * windowSize_h
                let RE_cir = UIBezierPath(arcCenter: CGPoint(x: RE_x, y: RE_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let RE_lay = CAShapeLayer()
                RE_lay.path = RE_cir.cgPath
                RE_lay.fillColor = UIColor.red.cgColor
                RE_lay.lineWidth = 3.0
                
                let RW_x = rightWrist.x * windowSize_w
                let RW_y = (1 - rightWrist.y) * windowSize_h
                let RW_cir = UIBezierPath(arcCenter: CGPoint(x: RW_x, y: RW_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let RW_lay = CAShapeLayer()
                RW_lay.path = RW_cir.cgPath
                RW_lay.fillColor = UIColor.red.cgColor
                RW_lay.lineWidth = 3.0
                
                let LS_x = leftShoulder.x * windowSize_w
                let LS_y = (1 - leftShoulder.y) * windowSize_h
                let LS_cir = UIBezierPath(arcCenter: CGPoint(x: LS_x, y: LS_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let LS_lay = CAShapeLayer()
                LS_lay.path = LS_cir.cgPath
                LS_lay.fillColor = UIColor.red.cgColor
                LS_lay.lineWidth = 3.0
                
                let LE_x = leftElbow.x * windowSize_w
                let LE_y = (1 - leftElbow.y) * windowSize_h
                let LE_cir = UIBezierPath(arcCenter: CGPoint(x: LE_x, y: LE_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let LE_lay = CAShapeLayer()
                LE_lay.path = LE_cir.cgPath
                LE_lay.fillColor = UIColor.red.cgColor
                LE_lay.lineWidth = 3.0
                
                let LW_x = leftWrist.x * windowSize_w
                let LW_y = (1 - leftWrist.y) * windowSize_h
                let LW_cir = UIBezierPath(arcCenter: CGPoint(x: LW_x, y: LW_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let LW_lay = CAShapeLayer()
                LW_lay.path = LW_cir.cgPath
                LW_lay.fillColor = UIColor.red.cgColor
                LW_lay.lineWidth = 3.0
                
                let RO_x = root.x * windowSize_w
                let RO_y = (1 - root.y) * windowSize_h
                let RO_cir = UIBezierPath(arcCenter: CGPoint(x: RO_x, y: RO_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let RO_lay = CAShapeLayer()
                RO_lay.path = RO_cir.cgPath
                RO_lay.fillColor = UIColor.red.cgColor
                RO_lay.lineWidth = 3.0
                
                let RH_x = rightHip.x * windowSize_w
                let RH_y = (1 - rightHip.y) * windowSize_h
                let RH_cir = UIBezierPath(arcCenter: CGPoint(x: RH_x, y: RH_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let RH_lay = CAShapeLayer()
                RH_lay.path = RH_cir.cgPath
                RH_lay.fillColor = UIColor.red.cgColor
                RH_lay.lineWidth = 3.0
                
                let RK_x = rightKnee.x * windowSize_w
                let RK_y = (1 - rightKnee.y) * windowSize_h
                let RK_cir = UIBezierPath(arcCenter: CGPoint(x: RK_x, y: RK_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let RK_lay = CAShapeLayer()
                RK_lay.path = RK_cir.cgPath
                RK_lay.fillColor = UIColor.red.cgColor
                RK_lay.lineWidth = 3.0
                
                let RA_x = rightAnkle.x * windowSize_w
                let RA_y = (1 - rightAnkle.y) * windowSize_h
                let RA_cir = UIBezierPath(arcCenter: CGPoint(x: RA_x, y: RA_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let RA_lay = CAShapeLayer()
                RA_lay.path = RA_cir.cgPath
                RA_lay.fillColor = UIColor.red.cgColor
                RA_lay.lineWidth = 3.0
                
                let LH_x = leftHip.x * windowSize_w
                let LH_y = (1 - leftHip.y) * windowSize_h
                let LH_cir = UIBezierPath(arcCenter: CGPoint(x: LH_x, y: LH_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let LH_lay = CAShapeLayer()
                LH_lay.path = LH_cir.cgPath
                LH_lay.fillColor = UIColor.red.cgColor
                LH_lay.lineWidth = 3.0
                
                let LA_x = leftAnkle.x * windowSize_w
                let LA_y = (1 - leftAnkle.y) * windowSize_h
                let LA_cir = UIBezierPath(arcCenter: CGPoint(x: LA_x, y: LA_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let LA_lay = CAShapeLayer()
                LA_lay.path = LA_cir.cgPath
                LA_lay.fillColor = UIColor.red.cgColor
                LA_lay.lineWidth = 3.0
                
                let LK_x = leftKnee.x * windowSize_w
                let LK_y = (1 - leftKnee.y) * windowSize_h
                let LK_cir = UIBezierPath(arcCenter: CGPoint(x: LK_x, y: LK_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
                let LK_lay = CAShapeLayer()
                LK_lay.path = LK_cir.cgPath
                LK_lay.fillColor = UIColor.red.cgColor
                LK_lay.lineWidth = 3.0
                
                // 描画する線をまとめるレイヤー
                let layer = CALayer()
                
                // 右肩から右肘
                let RStoRE_lay = CAShapeLayer()
                RStoRE_lay.lineWidth = 2.0
                RStoRE_lay.strokeColor = UIColor.green.cgColor
                let path1 = UIBezierPath()
                path1.move(to: CGPoint(x: RS_x, y: RS_y))
                path1.addLine(to: CGPoint(x: RE_x, y: RE_y))
                RStoRE_lay.path = path1.cgPath
                layer.addSublayer(RStoRE_lay)
                
                // 右肘から右手首
                let REtoRW_lay = CAShapeLayer()
                REtoRW_lay.lineWidth = 2.0
                REtoRW_lay.strokeColor = UIColor.green.cgColor
                let path2 = UIBezierPath()
                path2.move(to: CGPoint(x: RE_x, y: RE_y))
                path2.addLine(to: CGPoint(x: RW_x, y: RW_y))
                REtoRW_lay.path = path2.cgPath
                layer.addSublayer(REtoRW_lay)
                
                // 右肩から左肩
                let RStoLS_lay = CAShapeLayer()
                RStoLS_lay.lineWidth = 2.0
                RStoLS_lay.strokeColor = UIColor.green.cgColor
                let path3 = UIBezierPath()
                path3.move(to: CGPoint(x: RS_x, y: RS_y))
                path3.addLine(to: CGPoint(x: LS_x, y: LS_y))
                RStoLS_lay.path = path3.cgPath
                layer.addSublayer(RStoLS_lay)
                
                // 左肩から左肘
                let LStoLE_lay = CAShapeLayer()
                LStoLE_lay.lineWidth = 2.0
                LStoLE_lay.strokeColor = UIColor.green.cgColor
                let path4 = UIBezierPath()
                path4.move(to: CGPoint(x: LS_x, y: LS_y))
                path4.addLine(to: CGPoint(x: LE_x, y: LE_y))
                LStoLE_lay.path = path4.cgPath
                layer.addSublayer(LStoLE_lay)
                
                // 左肘から左手首
                let LEtoLW_lay = CAShapeLayer()
                LEtoLW_lay.lineWidth = 2.0
                LEtoLW_lay.strokeColor = UIColor.green.cgColor
                let path5 = UIBezierPath()
                path5.move(to: CGPoint(x: LE_x, y: LE_y))
                path5.addLine(to: CGPoint(x: LW_x, y: LW_y))
                LEtoLW_lay.path = path5.cgPath
                layer.addSublayer(LEtoLW_lay)
                
                // 左肩から左尻
                let LStoLH_lay = CAShapeLayer()
                LStoLH_lay.lineWidth = 2.0
                LStoLH_lay.strokeColor = UIColor.green.cgColor
                let path6 = UIBezierPath()
                path6.move(to: CGPoint(x: LS_x, y: LS_y))
                path6.addLine(to: CGPoint(x: LH_x, y: LH_y))
                LStoLH_lay.path = path6.cgPath
                layer.addSublayer(LStoLH_lay)
                
                // 右肩から右尻
                let RStoRH_lay = CAShapeLayer()
                RStoRH_lay.lineWidth = 2.0
                RStoRH_lay.strokeColor = UIColor.green.cgColor
                let path7 = UIBezierPath()
                path7.move(to: CGPoint(x: RS_x, y: RS_y))
                path7.addLine(to: CGPoint(x: RH_x, y: RH_y))
                RStoRH_lay.path = path7.cgPath
                layer.addSublayer(RStoRH_lay)
                
                // 右尻から右膝
                let RKtoRH_lay = CAShapeLayer()
                RKtoRH_lay.lineWidth = 2.0
                RKtoRH_lay.strokeColor = UIColor.green.cgColor
                let path8 = UIBezierPath()
                path8.move(to: CGPoint(x: RK_x, y: RK_y))
                path8.addLine(to: CGPoint(x: RH_x, y: RH_y))
                RKtoRH_lay.path = path8.cgPath
                layer.addSublayer(RKtoRH_lay)
                
                // 左尻から左膝
                let LKtoLH_lay = CAShapeLayer()
                LKtoLH_lay.lineWidth = 2.0
                LKtoLH_lay.strokeColor = UIColor.green.cgColor
                let path9 = UIBezierPath()
                path9.move(to: CGPoint(x: LK_x, y: LK_y))
                path9.addLine(to: CGPoint(x: LH_x, y: LH_y))
                LKtoLH_lay.path = path9.cgPath
                layer.addSublayer(LKtoLH_lay)
                
                // 右膝から右足首
                let RKtoRA_lay = CAShapeLayer()
                RKtoRA_lay.lineWidth = 2.0
                RKtoRA_lay.strokeColor = UIColor.green.cgColor
                let path10 = UIBezierPath()
                path10.move(to: CGPoint(x: RK_x, y: RK_y))
                path10.addLine(to: CGPoint(x: RA_x, y: RA_y))
                RKtoRA_lay.path = path10.cgPath
                layer.addSublayer(RKtoRA_lay)
                
                // 左膝から左足首
                let LKtoLA_lay = CAShapeLayer()
                LKtoLA_lay.lineWidth = 2.0
                LKtoLA_lay.strokeColor = UIColor.green.cgColor
                let path11 = UIBezierPath()
                path11.move(to: CGPoint(x: LK_x, y: LK_y))
                path11.addLine(to: CGPoint(x: LA_x, y: LA_y))
                LKtoLA_lay.path = path11.cgPath
                layer.addSublayer(LKtoLA_lay)
                
                // 右尻から左尻
                let RHtoLH_lay = CAShapeLayer()
                RHtoLH_lay.lineWidth = 2.0
                RHtoLH_lay.strokeColor = UIColor.green.cgColor
                let path12 = UIBezierPath()
                path12.move(to: CGPoint(x: RH_x, y: RH_y))
                path12.addLine(to: CGPoint(x: LH_x, y: LH_y))
                RHtoLH_lay.path = path12.cgPath
                layer.addSublayer(RHtoLH_lay)
                
                // 判定した画像に対して線を描画する
                let resultImage = drawShapeOnImage(image: image, shapeLayer: layer)
                
                //線付きの人物画像をアプリ内に保存していく　本来は画像を並べて動画に直す作業に持っていく
                let imageData = resultImage!.jpegData(compressionQuality: 1.0)
                let documentURL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0]
                let fileURL = documentURL.appendingPathComponent("frame\(frameNumber).jpg")
                try? imageData?.write(to: fileURL)
            }
        }catch{
            print("error")
        }
    }
```

ループ等を使えばもっと綺麗に描けるのかなと思いつつ、直書きでお恥ずかしいです。

# 3.4.5
```swift
// 動画を生成するファンクション　まず構造体を生成
    func createVideo(fps:Float) {
        struct dirImage {
            var URL: URL
            var name: String
            var image: UIImage
            
            init(URL: URL,  name: String, image: UIImage){
                self.URL = URL
                self.name = name
                self.image = image
            }
        }
        
        // 画像を読み込み、配列に入れる
        var imagesArray = [dirImage]()
        
        let fileManager = FileManager.default
        let documentsUrl = fileManager.urls(for: .documentDirectory, in: .userDomainMask)[0]
        let fileUrls = try! fileManager.contentsOfDirectory(at: documentsUrl, includingPropertiesForKeys: nil, options: [])
        
        for fileUrl in fileUrls {
            if let Frameimage = UIImage(contentsOfFile: fileUrl.path){
                imagesArray.append(dirImage(URL:fileUrl, name: fileUrl.lastPathComponent, image: Frameimage))
            }
        }
        // 名前順に入れ替え
        imagesArray = imagesArray.sorted { (name1, name2) -> Bool in
            let numericPart1 = name1.name.replacingOccurrences(of: "[^0-9]", with: "",options: .regularExpression)
            let numericPart2 = name2.name.replacingOccurrences(of: "[^0-9]", with: "",options: .regularExpression)
            guard let number1 = Int(numericPart1), let number2 = Int(numericPart2) else {
                return name1.name < name2.name
            }
            return number1 < number2
        }
        
        print(imagesArray)
        
        // 生成した動画を保存するパス
        let documentsPath = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)[0]
        as String
        let videoOutputPath = (documentsPath as NSString).appendingPathComponent("output.mp4")
        let videoWriter = try! AVAssetWriter(outputURL: URL(fileURLWithPath: videoOutputPath), fileType: .mp4)
        
        // 既にファイルがある場合は削除する
        // このコードを消したりつけたりする必要がある点を修正
        if fileManager.fileExists(atPath: documentsPath) {
            try! fileManager.removeItem(atPath: videoOutputPath)
        }
        
        // 最初の画像から動画のサイズを取得する
        let size = imagesArray.first?.image.size
        
        // 動画の保存先の指定、保存方法、動画のフレームを書き込むための準備
        guard let videoWriter = try? AVAssetWriter(outputURL: URL(fileURLWithPath: videoOutputPath), fileType: .mp4) else {
            abort()
        }
        let videoSettings = [
            AVVideoCodecKey: AVVideoCodecType.h264,
            AVVideoWidthKey: size?.width,
            AVVideoHeightKey: size?.height
        ] as [String : Any]
        
        let videoWriterInput = AVAssetWriterInput(mediaType: AVMediaType.video, outputSettings: videoSettings)
        videoWriterInput.expectsMediaDataInRealTime = true
        videoWriter.add(videoWriterInput)
        
        // 画像をフレームとして書き込むための用意
        let sourcePixcelBufferAttributesDictionary = [
            kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32ARGB),
            kCVPixelBufferWidthKey as String: size?.width,
            kCVPixelBufferHeightKey as String: size?.height
        ] as [String : Any]
        
        let adaptor = AVAssetWriterInputPixelBufferAdaptor(
            assetWriterInput: videoWriterInput,
            sourcePixelBufferAttributes: sourcePixcelBufferAttributesDictionary)
        videoWriterInput.expectsMediaDataInRealTime = true
        
        // 動画生成開始
        if (!videoWriter.startWriting()){
            print("Failed to start writing")
            return
        }
        
        videoWriter.startSession(atSourceTime: CMTime.zero)
        
        var frameCount: Int64 = 0
        // 各画像の表示する時間
        let durationForEachImage: Int64 = 1
        //let durationForEachImage: Float = 0.1
        //let fps: Int32 = 24
        let fps = fps
        
        // 構造体から要素を取り出す　[name, uiimage]
        for img in imagesArray {
            if !adaptor.assetWriterInput.isReadyForMoreMediaData {
                continue
            }
            // 取り出した要素の画像を選択しそれをcgImageに変換
            let cgImg = img.image.cgImage
            
            // 動画の時間を生成（その画像を表示する時間/開始時間と表示時間を渡す）
            //let frameTime = CMTimeMake(value: frameCount * Int64(fps) * durationForEachImage,timescale: fps)
            let frameTime = CMTimeMake(value: frameCount * 1, timescale: Int32(fps))
            let second = CMTimeGetSeconds(frameTime)
            print(second)
            guard let buffer = pixelBuffer(for: cgImg) else{
                continue
            }
            if !adaptor.append(buffer, withPresentationTime: frameTime) {
                print("Failed to append buffer. [image : \(cgImg)]")
            }
            frameCount += 1
        }
        
        videoWriterInput.markAsFinished()
        //videoWriter.endSession(atSourceTime: CMTimeMake(value: frameCount * Int64(fps) * durationForEachImage, timescale: fps))
        videoWriter.endSession(atSourceTime: CMTimeMake(value: frameCount * 1, timescale: Int32(fps)))
        videoWriter.finishWriting {
            print("Finish writing")
            self.deleteFIles(withExtension: "jpg", inDirectory: documentsPath)
        }
    }
```

補足
構造体を使用した理由としては
ドキュメント内に保存したファイルを名前順並び替える為です。

出力した動画ファイルを削除するコードがありますが、既に動画ファイルがある場合とない場合でコメントアウトしたり、しなかったりしないと挙動がおかしくなります。
（修正ポイント）