#WWDC19

Creating Great Apps Using
Core ML and ARKit••Brent Dimick, Core ML
Scott Gauthreaux, Core ML© 2019 Apple Inc. All rights reserved. Redistribution or public display not permitted without written permission from Apple.

+Can Machine Learning help?

Understanding your modelInput
299 px299 pxInputOutput299 pxSkier93%

Snowboarder 7%299 pxOur Journey•

Recognizing diceProgrammaticallyRecognizing Dice

Recognizing Dice

Recognizing Dice

Recognizing Dice

Recognizing Dice

Using Machine LearningDice•DemoScott Gauthreaux, Core ML// Handling object detection observations
func handleObservations(_ observations: [VNRecognizedObjectObservation]) {self.diceCount = observations.countfor observation in observations {let objectBounds = self.bounds(for: observation)
let shapeLayer = self.createRoundedRectLayerWithBounds(objectBounds)
self.detectionOverlay.addSublayer(shapeLayer)}}// Handling object detection observations
func handleObservations(_ observations: [VNRecognizedObjectObservation]) {self.diceCount = observations.countfor observation in observations {let objectBounds = self.bounds(for: observation)
let shapeLayer = self.createRoundedRectLayerWithBounds(objectBounds)
self.detectionOverlay.addSublayer(shapeLayer)}}// Handling object detection observations
func handleObservations(_ observations: [VNRecognizedObjectObservation]) {self.diceCount = observations.countfor observation in observations {let objectBounds = self.bounds(for: observation)
let shapeLayer = self.createRoundedRectLayerWithBounds(objectBounds)
self.detectionOverlay.addSublayer(shapeLayer)}}// Handling object detection observations
func handleObservations(_ observations: [VNRecognizedObjectObservation]) {self.diceCount = observations.countfor observation in observations {let objectBounds = self.bounds(for: observation)
let shapeLayer = self.createRoundedRectLayerWithBounds(objectBounds)
self.detectionOverlay.addSublayer(shapeLayer)}}// Handling object detection observations
func handleObservations(_ observations: [VNRecognizedObjectObservation]) {self.diceCount = observations.countfor observation in observations {let objectBounds = self.bounds(for: observation)
let shapeLayer = self.createRoundedRectLayerWithBounds(objectBounds)
self.detectionOverlay.addSublayer(shapeLayer)}}

One step further.
Recognizing the values on the dice.

51

Data Collection and Annotation
51

4

5
5

4

Focusing on Tops
51

455

44

5
56

613

163

•

DemoWhen Does a Roll End?

When Does a Roll End?

What we observe

When Does a Roll End?

What we observe
Interpreting model output

// Function to tell if a roll has ended with the given values
func hasRollEnded(newObservations: [VNRecognizedObjectObservation],
oldObservations: [VNRecognizedObjectObservation]) -> Bool {if oldObservations.count != newObservations.count {return false}
var matches = 0
for new in newObservations {for old in oldObservations {if new.labels.first?.identifier == old.labels.first?.identifier &&overlapRatio(rectA: old.boundingBox, rectB: new.boundingBox) > 0.85 {matches += 1}}}
return matches == newObservations.count}// Function to tell if a roll has ended with the given values
func hasRollEnded(newObservations: [VNRecognizedObjectObservation],
oldObservations: [VNRecognizedObjectObservation]) -> Bool {if oldObservations.count != newObservations.count {return false}
var matches = 0
for new in newObservations {for old in oldObservations {if new.labels.first?.identifier == old.labels.first?.identifier &&overlapRatio(rectA: old.boundingBox, rectB: new.boundingBox) > 0.85 {matches += 1}}}
return matches == newObservations.count}// Function to tell if a roll has ended with the given values
func hasRollEnded(newObservations: [VNRecognizedObjectObservation],oldObservations: [VNRecognizedObjectObservation]) -> Bool {if oldObservations.count != newObservations.count {return false}
var matches = 0for new in newObservations {for old in oldObservations {if new.labels.first?.identifier == old.labels.first?.identifier &&overlapRatio(rectA: old.boundingBox, rectB: new.boundingBox) > 0.85 {matches += 1}}}
return matches == newObservations.count}// Function to tell if a roll has ended with the given values
func hasRollEnded(newObservations: [VNRecognizedObjectObservation],
oldObservations: [VNRecognizedObjectObservation]) -> Bool {if oldObservations.count != newObservations.count {return false}var matches = 0
for new in newObservations {for old in oldObservations {if new.labels.first?.identifier == old.labels.first?.identifier &&overlapRatio(rectA: old.boundingBox, rectB: new.boundingBox) > 0.85 {matches += 1}}}
return matches == newObservations.count}// Function to tell if a roll has ended with the given values
func hasRollEnded(newObservations: [VNRecognizedObjectObservation],
oldObservations: [VNRecognizedObjectObservation]) -> Bool {if oldObservations.count != newObservations.count {return false}
var matches = 0
for new in newObservations {for old in oldObservations {if new.labels.first?.identifier == old.labels.first?.identifier &&overlapRatio(rectA: old.boundingBox, rectB: new.boundingBox) > 0.85 {matches += 1}}}
return matches == newObservations.count}•

Recognizing dice

•

Recognizing dice•Handling input

// Turning a drawing into a number with PencilKit and Vision
let model = try VNCoreMLModel(for: MNISTClassifier().model)
let request = VNCoreMLRequest(model: model)
request.imageCropAndScaleOption = .aspectFit
let image = canvasView.drawing.image(from: canvasView.drawing.bounds, scale: 1.0)
guard let cgImage = image.cgImage,requestHandler = VNImageRequestHandler(cgImage: cgImage, options: [:]) else {return}
try requestHandler.perform([request])
let results = request.results as? [VNClassificationObservation]

// Turning a drawing into a number with PencilKit and Vision
let model = try VNCoreMLModel(for: MNISTClassifier().model)
let request = VNCoreMLRequest(model: model)
request.imageCropAndScaleOption = .aspectFit
let image = canvasView.drawing.image(from: canvasView.drawing.bounds, scale: 1.0)
guard let cgImage = image.cgImage,requestHandler = VNImageRequestHandler(cgImage: cgImage, options: [:]) else {return}
try requestHandler.perform([request])
let results = request.results as? [VNClassificationObservation]

// Turning a drawing into a number with PencilKit and Vision
let model = try VNCoreMLModel(for: MNISTClassifier().model)
let request = VNCoreMLRequest(model: model)
request.imageCropAndScaleOption = .aspectFit
let image = canvasView.drawing.image(from: canvasView.drawing.bounds, scale: 1.0)
guard let cgImage = image.cgImage,requestHandler = VNImageRequestHandler(cgImage: cgImage, options: [:]) else {return}
try requestHandler.perform([request])
let results = request.results as? [VNClassificationObservation]NEW// Turning a drawing into a number with PencilKit and Vision
let model = try VNCoreMLModel(for: MNISTClassifier().model)
let request = VNCoreMLRequest(model: model)
request.imageCropAndScaleOption = .aspectFit
let image = canvasView.drawing.image(from: canvasView.drawing.bounds, scale: 1.0)
guard let cgImage = image.cgImage,requestHandler = VNImageRequestHandler(cgImage: cgImage, options: [:]) else {return}
try requestHandler.perform([request])
let results = request.results as? [VNClassificationObservation]

// Turning a drawing into a number with PencilKit and Vision
let model = try VNCoreMLModel(for: MNISTClassifier().model)
let request = VNCoreMLRequest(model: model)
request.imageCropAndScaleOption = .aspectFit
let image = canvasView.drawing.image(from: canvasView.drawing.bounds, scale: 1.0)
guard let cgImage = image.cgImage,requestHandler = VNImageRequestHandler(cgImage: cgImage, options: [:]) else {return}
try requestHandler.perform([request])
let results = request.results as? [VNClassificationObservation]

1 2 3 5 9Predicted

Predicted

1

DownscaledDownscaled

Predicted

7

NEW

canvasView.allowsFingerDrawing = true
canvasView.tool = PKInkingTool(.marker, color: .white, width: 60.0)

NEWcanvasView.allowsFingerDrawing = true
canvasView.tool = PKInkingTool(.marker, color: .white, width: 60.0)

NEWcanvasView.allowsFingerDrawing = true
canvasView.tool = PKInkingTool(.marker, color: .white, width: 60.0)•

•

Demo

NEW

// How to enable offline Speech to Text
let speechRecognitionRequest = SFSpeechAudioBufferRecognitionRequest()
speechRecognitionRequest.requiresOnDeviceRecognition = true

NEW// How to enable offline Speech to Text
let speechRecognitionRequest = SFSpeechAudioBufferRecognitionRequest()
speechRecognitionRequest.requiresOnDeviceRecognition = true•

Recognizing dice•Handling input•

Recognizing dice•Handling input•

Finalizing the game

+•Rules of the Game

182

7

3
654

182

7

3
654

182

7

3
654

182

7

3
654

18

2

73
654

5 + 1 = 6

5 - 1 = 4

••DemoSummary

Recognizing diceHandling input
Finalizing the game

More Informationdeveloper.apple.com/wwdc19/228

Machine Learning Lab

Friday, 2:00