Issue 1:
Describe the bug

This bug seems to be related to the bug #17701 but can be reproduced by a completely different approach.

Three will throw an error when trying to apply a outline pass on a scene with instanced and normal meshes.

Error:

three.module.js:12456 Uncaught TypeError: Cannot read property 'isInterleavedBufferAttribute' of undefined
at Object.get (three.module.js:12456)
at setupVertexAttributes (three.module.js:14119)
at Object.setup (three.module.js:13789)
at WebGLRenderer.renderBufferDirect (three.module.js:23669)
at renderObject (three.module.js:24192)
at renderObjects (three.module.js:24165)
at WebGLRenderer.render (three.module.js:23959)
at OutlinePass.render (OutlinePass.js:319)
at EffectComposer.render (EffectComposer.js:150)
at init (index.js:108)

To Reproduce

Steps to reproduce the behavior:

Add instanced mesh to the scene

Add normal mesh to scene

Add normal mesh to the scene

Try to apply an outline on the first normal mesh

Code

import {
DoubleSide,
InstancedMesh,
Matrix4,
Mesh,
MeshLambertMaterial,
MeshPhongMaterial,
PerspectiveCamera,
PlaneBufferGeometry,
Quaternion,
Scene,
SphereBufferGeometry,
TorusBufferGeometry,
Vector2,
Vector3,
WebGLRenderer
} from "three";
import {EffectComposer} from "three/examples/jsm/postprocessing/EffectComposer";
import {RenderPass} from "three/examples/jsm/postprocessing/RenderPass";
import {OutlinePass} from "three/examples/jsm/postprocessing/OutlinePass";

// issue related to https://github.com/mrdoob/three.js/issues/17701
// code based on https://github.com/mrdoob/three.js/blob/master/examples/webgl_postprocessing_outline.html

function addSpheresInstanced() {
const geometry = new SphereBufferGeometry(3, 48, 24);
const material = new MeshLambertMaterial({color: 0xff00aa});

const instanced = new InstancedMesh(geometry, material, 10);

const mat = new Matrix4();

const position = new Vector3();
const rotation = new Quaternion();
const scale = new Vector3();

for (let i = 0; i < 10; i++) {
position.set(Math.random() * 4 - 2, Math.random() * 4 - 2, Math.random() * 4 - 2);

const scaleAmount = Math.random() * 0.3 + 0.1;
scale.set(scaleAmount, scaleAmount, scaleAmount);

mat.compose(position, rotation, scale);
instanced.setMatrixAt(i, mat);
}

instanced.instanceMatrix.needsUpdate = true;

return instanced;
}

function addFloor() {
const floorGeometry = new PlaneBufferGeometry(12, 12);
const floorMaterial = new MeshLambertMaterial({side: DoubleSide});

const floorMesh = new Mesh(floorGeometry, floorMaterial);
floorMesh.rotation.x -= Math.PI * 0.5;
floorMesh.position.y -= 1.5;
floorMesh.receiveShadow = true;

return floorMesh;
}

function addTorus() {
const torusGeometry = new TorusBufferGeometry(1, 0.3, 16, 100);
const torusMaterial = new MeshPhongMaterial({color: 0xffaaff});

const torusMesh = new Mesh(torusGeometry, torusMaterial);
torusMesh.position.z = -4;
torusMesh.receiveShadow = true;
torusMesh.castShadow = true;

return torusMesh;
}

function init() {
const width = window.innerWidth;
const height = window.innerHeight;

const renderer = new WebGLRenderer();
renderer.setSize(width, height);
document.body.appendChild(renderer.domElement);

const scene = new Scene();

const camera = new PerspectiveCamera(45, width / height, 0.1, 100);
camera.position.set(0, 0, 8);

const spheresInstanced = addSpheresInstanced();
scene.add(spheresInstanced);

const torus = addTorus();
scene.add(torus);

const floor = addFloor();
scene.add(floor);

const composer = new EffectComposer(renderer);

const renderPass = new RenderPass(scene, camera);
composer.addPass(renderPass);

const outlinePass = new OutlinePass(new Vector2(width, height), scene, camera);
composer.addPass(outlinePass);

outlinePass.selectedObjects = [torus];

composer.render();
}

init();

Live example

https://jsfiddle.net/q8e0h7du/

Expected behavior

There should be no error and the outline should be rendered around the torus.

Platform:

Device: Desktop

OS: Windows

Browser: Chrome

Three.js version: r125

Issue 2:
Description of the problem

OutlinePass does not appear to properly support InstancedMesh. I've created an example here that uses the same setup of OutlinePass in the example in the docs. The sphere is a regular Mesh and the squares are the instances of InstancedMesh.

https://jsfiddle.net/anthonysimone/dL8s2qmv/

It seems like there's a couple things going on here.

If you hover over the regular mesh first (sphere), then one of the instanced mesh instances, the base position of the InstancedMesh is outlined any time you’re hovering over any of the instances. In the above example all of the instances have been moved from the default position to demonstrate this clearly.

If you first hover over any of the box instances, the entire collection of instances (though not the base position) of the InstancedMesh are outlined, but then hovering over a regular mesh (the sphere) causes an error.

I dug into OutlinePass a bit and it seems like it might just not be taking into account handling instanced meshes separately from regular meshes. I'm still pretty new to Three.js, so I don't have a potential fix to suggest at this point.

Three.js version

r113

Browser

All of them

Chrome

Firefox

Internet Explorer

OS

All of them

Windows

macOS

Linux

Android

iOS