three.ar.js
Version:
A helper three.js library for building AR web experiences that run in WebARonARKit and WebARonARCore
421 lines (377 loc) • 12.6 kB
JavaScript
/*
* Copyright 2017 Google Inc. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the 'License');
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an 'AS IS' BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { isARKit } from './ARUtils';
import vertexSource from './shaders/arview.vert';
import fragmentSource from './shaders/arview.frag';
import fragmentSourceOES from './shaders/arview-oes.frag';
import preserveGLState from 'gl-preserve-state';
/**
* Creates and load a shader from a string, type specifies either 'vertex' or 'fragment'
*
* @param {WebGLRenderingContext} gl
* @param {string} str
* @param {string} type
* @return {!WebGLShader}
*/
function getShader(gl, str, type) {
let shader;
if (type == 'fragment') {
shader = gl.createShader(gl.FRAGMENT_SHADER);
} else if (type == 'vertex') {
shader = gl.createShader(gl.VERTEX_SHADER);
} else {
return null;
}
gl.shaderSource(shader, str);
gl.compileShader(shader);
const result = gl.getShaderParameter(shader, gl.COMPILE_STATUS);
if (!result) {
alert(gl.getShaderInfoLog(shader));
return null;
}
return shader;
}
/**
* Creates a shader program from vertex and fragment shader sources
*
* @param {WebGLRenderingContext} gl
* @param {string} vs
* @param {string} fs
* @return {!WebGLProgram}
*/
function getProgram(gl, vs, fs) {
const vertexShader = getShader(gl, vs, 'vertex');
const fragmentShader = getShader(gl, fs, 'fragment');
if (!fragmentShader) {
return null;
}
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, vertexShader);
gl.attachShader(shaderProgram, fragmentShader);
gl.linkProgram(shaderProgram);
const result = gl.getProgramParameter(shaderProgram, gl.LINK_STATUS);
if (!result) {
alert('Could not initialise arview shaders');
}
return shaderProgram;
}
/**
* Calculate the correct orientation depending on the device and the camera
* orientations.
*
* @param {number} screenOrientation
* @param {number} seeThroughCameraOrientation
* @return {number}
*/
function combineOrientations(screenOrientation, seeThroughCameraOrientation) {
let seeThroughCameraOrientationIndex = 0;
switch (seeThroughCameraOrientation) {
case 90:
seeThroughCameraOrientationIndex = 1;
break;
case 180:
seeThroughCameraOrientationIndex = 2;
break;
case 270:
seeThroughCameraOrientationIndex = 3;
break;
default:
seeThroughCameraOrientationIndex = 0;
break;
}
let screenOrientationIndex = 0;
switch (screenOrientation) {
case 90:
screenOrientationIndex = 1;
break;
case 180:
screenOrientationIndex = 2;
break;
case 270:
screenOrientationIndex = 3;
break;
default:
screenOrientationIndex = 0;
break;
}
let ret = screenOrientationIndex - seeThroughCameraOrientationIndex;
if (ret < 0) {
ret += 4;
}
return ret % 4;
}
/**
* Renders the ar camera's video texture
*/
class ARVideoRenderer {
/**
* @param {VRDisplay} vrDisplay
* @param {WebGLRenderingContext} gl
*/
constructor(vrDisplay, gl) {
this.vrDisplay = vrDisplay;
this.gl = gl;
if (this.vrDisplay) {
this.passThroughCamera = vrDisplay.getPassThroughCamera();
// Depending on the type of passThroughCamera, use a different
// target for the texture and different shaders.
if (this.passThroughCamera instanceof Image) {
this.textureTarget = gl.TEXTURE_2D;
this.fragmentSource = fragmentSource;
} else {
this.textureTarget = gl.TEXTURE_EXTERNAL_OES;
this.fragmentSource = fragmentSourceOES;
}
this.program = getProgram(gl, vertexSource, this.fragmentSource);
}
gl.useProgram(this.program);
// Setup a quad
this.vertexPositionAttribute = gl.getAttribLocation(
this.program,
'aVertexPosition'
);
this.textureCoordAttribute = gl.getAttribLocation(
this.program,
'aTextureCoord'
);
this.samplerUniform = gl.getUniformLocation(this.program, 'uSampler');
this.vertexPositionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, this.vertexPositionBuffer);
let vertices = [
-1.0,
1.0,
0.0,
-1.0,
-1.0,
0.0,
1.0,
1.0,
0.0,
1.0,
-1.0,
0.0,
];
let f32Vertices = new Float32Array(vertices);
gl.bufferData(gl.ARRAY_BUFFER, f32Vertices, gl.STATIC_DRAW);
this.vertexPositionBuffer.itemSize = 3;
this.vertexPositionBuffer.numItems = 12;
this.textureCoordBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, this.textureCoordBuffer);
// Precalculate different texture UV coordinates depending on the possible
// orientations of the device depending if there is a VRDisplay or not
let textureCoords = null;
if (this.vrDisplay) {
// In the case of ARKit camera frame the canvas might not have been
// created of the correct size so the UV values can't be calculated from it.
let u = window.WebARonARKitSendsCameraFrames ? 1.0 :
this.passThroughCamera.width / this.passThroughCamera.textureWidth;
let v = window.WebARonARKitSendsCameraFrames ? 1.0 :
this.passThroughCamera.height / this.passThroughCamera.textureHeight;
textureCoords = [
[0.0, 0.0, 0.0, v, u, 0.0, u, v],
[u, 0.0, 0.0, 0.0, u, v, 0.0, v],
[u, v, u, 0.0, 0.0, v, 0.0, 0.0],
[0.0, v, u, v, 0.0, 0.0, u, 0.0],
];
} else {
textureCoords = [
[0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0],
[1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0],
[1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0],
[0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0],
];
}
this.f32TextureCoords = [];
for (let i = 0; i < textureCoords.length; i++) {
this.f32TextureCoords.push(new Float32Array(textureCoords[i]));
}
// Store the current combined orientation to check if it has changed
// during the update calls and use the correct texture coordinates.
this.combinedOrientation = combineOrientations(
screen.orientation ? screen.orientation.angle : window.orientation,
this.passThroughCamera.orientation
);
gl.bufferData(
gl.ARRAY_BUFFER,
this.f32TextureCoords[this.combinedOrientation],
gl.STATIC_DRAW
);
this.textureCoordBuffer.itemSize = 2;
this.textureCoordBuffer.numItems = 8;
gl.bindBuffer(gl.ARRAY_BUFFER, null);
this.indexBuffer = gl.createBuffer();
gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this.indexBuffer);
let indices = [0, 1, 2, 2, 1, 3];
let ui16Indices = new Uint16Array(indices);
gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, ui16Indices, gl.STATIC_DRAW);
this.indexBuffer.itemSize = 1;
this.indexBuffer.numItems = 6;
gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, null);
this.texture = gl.createTexture();
gl.useProgram(null);
// The projection matrix will be based on an identify orthographic camera
this.projectionMatrix = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1];
this.mvMatrix = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1];
return this;
}
/**
* Renders the quad
*/
render() {
let gl = this.gl;
const bindings = [
gl.ARRAY_BUFFER_BINDING,
gl.ELEMENT_ARRAY_BUFFER_BINDING,
gl.CURRENT_PROGRAM,
gl.TEXTURE_BINDING_2D,
];
preserveGLState(gl, bindings, () => {
// If the camera pass through is still not valid, skip the rendering.
if (this.passThroughCamera.textureWidth === 0 ||
this.passThroughCamera.textureHeight === 0) {
return;
}
// Save and configure values we need.
let previousFlipY = gl.getParameter(gl.UNPACK_FLIP_Y_WEBGL);
let previousWinding = gl.getParameter(gl.FRONT_FACE);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
gl.frontFace(gl.CCW);
gl.useProgram(this.program);
gl.bindBuffer(gl.ARRAY_BUFFER, this.vertexPositionBuffer);
gl.enableVertexAttribArray(this.vertexPositionAttribute);
gl.vertexAttribPointer(
this.vertexPositionAttribute,
this.vertexPositionBuffer.itemSize,
gl.FLOAT,
false,
0,
0
);
gl.bindBuffer(gl.ARRAY_BUFFER, this.textureCoordBuffer);
// Check the current orientation of the device combined with the
// orientation of the VRSeeThroughCamera to determine the correct UV
// coordinates to be used.
let combinedOrientation = combineOrientations(
screen.orientation ? screen.orientation.angle : window.orientation,
this.passThroughCamera.orientation
);
if (combinedOrientation !== this.combinedOrientation) {
this.combinedOrientation = combinedOrientation;
gl.bufferData(
gl.ARRAY_BUFFER,
this.f32TextureCoords[this.combinedOrientation],
gl.STATIC_DRAW
);
}
gl.enableVertexAttribArray(this.textureCoordAttribute);
gl.vertexAttribPointer(
this.textureCoordAttribute,
this.textureCoordBuffer.itemSize,
gl.FLOAT,
false,
0,
0
);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(this.textureTarget, this.texture);
// Update the content of the texture in every frame.
gl.texImage2D(
this.textureTarget,
0,
gl.RGB,
gl.RGB,
gl.UNSIGNED_BYTE,
this.passThroughCamera
);
gl.uniform1i(this.samplerUniform, 0);
// The texture from ARKit is not power of 2 friendly so these parameters
// are needed.
if (window.WebARonARKitSendsCameraFrames) {
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
}
gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this.indexBuffer);
gl.drawElements(
gl.TRIANGLES,
this.indexBuffer.numItems,
gl.UNSIGNED_SHORT,
0
);
// Restore previous values.
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, previousFlipY);
gl.frontFace(previousWinding);
});
}
}
/**
* A helper class that takes a VRDisplay with AR capabilities
* and renders the see through camera to the passed in WebGLRenderer's
* context.
*/
class ARView {
/**
* @param {VRDisplay} vrDisplay
* @param {THREE.WebGLRenderer} renderer
*/
constructor(vrDisplay, renderer) {
this.vrDisplay = vrDisplay;
// Only with ARKit and the camera frames are not being passed, we should
// not do anything and let the transparent webview show the camera
// underneath.
if (isARKit(this.vrDisplay) && !window.WebARonARKitSendsCameraFrames) {
return;
}
this.renderer = renderer;
this.gl = renderer.context;
this.videoRenderer = new ARVideoRenderer(vrDisplay, this.gl);
// Cache the width/height so we're not potentially forcing
// a reflow if there's been a style invalidation
this.width = window.innerWidth;
this.height = window.innerHeight;
window.addEventListener('resize', this.onWindowResize.bind(this), false);
}
/**
* Updates the stored width/height of window on resize.
*/
onWindowResize() {
this.width = window.innerWidth;
this.height = window.innerHeight;
}
/**
* Renders the see through camera to the passed in renderer
*/
render() {
// Only with ARKit and the camera frames are not being passed, we should
// not do anything and let the transparent webview show the camera
// underneath.
if (isARKit(this.vrDisplay) && !window.WebARonARKitSendsCameraFrames) {
return;
}
let gl = this.gl;
let dpr = window.devicePixelRatio;
let width = this.width * dpr;
let height = this.height * dpr;
if (gl.viewportWidth !== width) {
gl.viewportWidth = width;
}
if (gl.viewportHeight !== height) {
gl.viewportHeight = height;
}
this.gl.viewport(0, 0, gl.viewportWidth, gl.viewportHeight);
this.videoRenderer.render();
}
}
export default ARView;