UNPKG

gl-chromakey

Version:

Chroma key a video/image/canvas element in real time using the GPU

551 lines (526 loc) 22.8 kB
const _ = `#version 300 es precision mediump float; in vec2 vTexCoord; out vec4 pixel; uniform sampler2D source; uniform sampler2D alpha; // Standard BT.709 RGB to YUV conversion vec3 RGBtoYUV(vec3 rgb) { return vec3( rgb.r * 0.2126 + rgb.g * 0.7152 + rgb.b * 0.0722, // Y (luma) rgb.r * -0.1146 + rgb.g * -0.3854 + rgb.b * 0.5, // U (chroma) rgb.r * 0.5 + rgb.g * -0.4542 + rgb.b * -0.0458 // V (chroma) ); } vec4 ProcessChromaKey(vec2 texCoord, vec3 keyColor, float tolerance, float smoothness, float spill) { // Remap 0-1 parameters to effective ranges float mappedTolerance = tolerance * 0.2; // 0-0.2 range float mappedSmoothness = smoothness * 0.2 + 0.01; // 0.01-0.21 range float mappedSpill = spill * 0.5 + 0.01; // 0.01-0.51 range vec4 rgba = texture(source, texCoord); // Convert to YUV for both pixel and key color vec3 pixelYUV = RGBtoYUV(rgba.rgb); vec3 keyYUV = RGBtoYUV(keyColor); // Use improved distance calculation that includes luminance for achromatic colors vec2 pixelUV = pixelYUV.yz; vec2 keyUV = keyYUV.yz; float chromaDist = distance(pixelUV, keyUV); // For achromatic colors (low chrominance), also consider luminance difference float chromaLength = length(keyUV); if (chromaLength < 0.1) { // Key color is achromatic (black, white, gray) float lumaDiff = abs(pixelYUV.x - keyYUV.x); chromaDist = max(chromaDist, lumaDiff); } float baseMask = chromaDist - mappedTolerance; float fullMask = pow(clamp(max(baseMask, 0.0) / mappedSmoothness, 0., 1.), 1.5); float spillVal = pow(clamp(max(baseMask, 0.0) / mappedSpill, 0., 1.), 1.5); rgba.a = fullMask; // Use the Y component from YUV for more accurate desaturation vec3 yuv = RGBtoYUV(rgba.rgb); float desat = yuv.x; // Use luma component instead of manual calculation rgba.rgb = mix(vec3(desat), rgba.rgb, spillVal); return rgba; } // sample each corner's RGB value vec3[4] corners (void) { ivec2 size = textureSize(alpha, 0); vec3 p[4]; p[0] = vec3(texelFetch(alpha, ivec2(0, 0), 0)); p[1] = vec3(texelFetch(alpha, ivec2(size.x - 1, 0), 0)); p[2] = vec3(texelFetch(alpha, ivec2(0, size.y - 1), 0)); p[3] = vec3(texelFetch(alpha, size - 1, 0)); return p; } // average the two "nearest" colors vec3 auto (void) { vec3 p[4] = corners(); float minDist = 999.0; int idx1 = 0, idx2 = 1; // Find the two closest corner colors for (int i = 0; i < 4; i++) { for (int j = i + 1; j < 4; j++) { float dist = distance(p[i], p[j]); if (dist < minDist) { minDist = dist; idx1 = i; idx2 = j; } } } return (p[idx1] + p[idx2]) / 2.0; } // show corner pixels from downsampled source void debug (void) { vec3 p[4] = corners(); if (vTexCoord.x < 0.1) { if (vTexCoord.y < 0.1) { pixel = vec4(p[0], 1); // top left corner } else if (vTexCoord.y > 0.9) { pixel = vec4(p[2], 1); // bottom left corner } } else if (vTexCoord.x > 0.9) { if (vTexCoord.y < 0.1) { pixel = vec4(p[1], 1); // top right corner } else if (vTexCoord.y > 0.9) { pixel = vec4(p[3], 1); // bottom right corner } } } void main(void) { pixel = texture(source, vTexCoord); %keys% } `, E = `#version 300 es precision mediump float; in vec3 position; in vec2 texCoord; out vec2 vTexCoord; void main(void) { gl_Position = vec4(position, 1.0); vTexCoord = vec2(texCoord.s, texCoord.t); } `; class p { gl; vertexShader; fragmentShader; program; uniforms; attributes; location_position; location_texCoord; set_source; set_alpha; constructor(e, a, t) { this.gl = e, this.vertexShader = this.compileShader(a, !1), this.fragmentShader = this.compileShader(t, !0), this.program = e.createProgram(); let r = ""; if (!this.program || !this.vertexShader || !this.fragmentShader) throw new Error("Failed to create shader program or shaders"); e.attachShader(this.program, this.vertexShader); let i = e.getShaderInfoLog(this.vertexShader); if (i && (r += `Vertex shader error: ${i} `), e.attachShader(this.program, this.fragmentShader), i = e.getShaderInfoLog(this.fragmentShader), i && (r += `Fragment shader error: ${i} `), e.linkProgram(this.program), !e.getProgramParameter(this.program, e.LINK_STATUS)) throw r += e.getProgramInfoLog(this.program), e.deleteProgram(this.program), e.deleteShader(this.vertexShader), e.deleteShader(this.fragmentShader), new Error(`Could not initialise shader: ${r}`); e.useProgram(this.program); const s = e.getProgramParameter(this.program, e.ACTIVE_UNIFORMS); this.uniforms = []; let o, n, h, d; for (o = 0; o < s; ++o) { if (n = e.getActiveUniform(this.program, o), !n || (h = n.name, d = e.getUniformLocation(this.program, h), !d)) continue; const l = n; d.name = h, l.set = this[`set_${h}`] = this.makeShaderSetter(l, d), l.get = this[`get_${h}`] = this.makeShaderGetter(d), l.loc = this[`location_${h}`] = d, this.uniforms.push(l); } const m = e.getProgramParameter(this.program, e.ACTIVE_ATTRIBUTES); for (this.attributes = [], o = 0; o < m; ++o) n = e.getActiveAttrib(this.program, o), n && (h = n.name, d = e.getAttribLocation(this.program, h), this[`location_${h}`] = d, this.attributes.push(h)); } compileShader(e, a) { const { gl: t } = this, r = a ? t.createShader(t.FRAGMENT_SHADER) : t.createShader(t.VERTEX_SHADER); if (!r) throw new Error("Failed to create shader"); if (t.shaderSource(r, e), t.compileShader(r), !t.getShaderParameter(r, t.COMPILE_STATUS)) throw new Error(`Shader error: ${t.getShaderInfoLog(r)}`); return r; } makeShaderSetter(e, a) { const t = this.gl; switch (e.type) { case t.SAMPLER_2D: return (r) => { e.glTexture = t[`TEXTURE${r}`], t.uniform1i(a, r); }; case t.BOOL: case t.INT: return (r) => { t.uniform1i(a, r); }; case t.FLOAT: return (r) => { t.uniform1f(a, r); }; case t.FLOAT_VEC2: return (r, i) => { t.uniform2f(a, r, i); }; case t.FLOAT_VEC3: return (r, i, s) => { t.uniform3f(a, r, i, s); }; case t.FLOAT_VEC4: return (r, i, s, o) => { t.uniform4f(a, r, i, s, o); }; case t.FLOAT_MAT3: return (r) => { t.uniformMatrix3fv(a, !1, r); }; case t.FLOAT_MAT4: return (r) => { t.uniformMatrix4fv(a, !1, r); }; } return () => { throw new Error(`ShaderProgram doesn't know how to set type: ${e.type}`); }; } makeShaderGetter(e) { return () => this.gl.getUniform(this.program, e); } useProgram() { this.gl.useProgram(this.program); } unload() { this.gl.deleteShader(this.vertexShader), this.gl.deleteShader(this.fragmentShader), this.gl.deleteProgram(this.program); } } class T { gl; format; framebuffer; renderbuffer; texture; constructor(e, a, t, r = e.UNSIGNED_BYTE) { if (this.gl = e, this.format = r, this.framebuffer = e.createFramebuffer(), e.bindFramebuffer(e.FRAMEBUFFER, this.framebuffer), this.renderbuffer = e.createRenderbuffer(), this.texture = e.createTexture(), e.bindTexture(e.TEXTURE_2D, this.texture), e.texParameteri(e.TEXTURE_2D, e.TEXTURE_MAG_FILTER, e.LINEAR), e.texParameteri(e.TEXTURE_2D, e.TEXTURE_MIN_FILTER, e.LINEAR), e.texParameteri(e.TEXTURE_2D, e.TEXTURE_WRAP_S, e.CLAMP_TO_EDGE), e.texParameteri(e.TEXTURE_2D, e.TEXTURE_WRAP_T, e.CLAMP_TO_EDGE), this.setSize(a, t), e.framebufferTexture2D(e.FRAMEBUFFER, e.COLOR_ATTACHMENT0, e.TEXTURE_2D, this.texture, 0), e.bindFramebuffer(e.FRAMEBUFFER, null), !e.isFramebuffer(this.framebuffer)) throw new Error("Invalid framebuffer"); const i = e.checkFramebufferStatus(e.FRAMEBUFFER); switch (i) { case e.FRAMEBUFFER_COMPLETE: break; case e.FRAMEBUFFER_INCOMPLETE_ATTACHMENT: throw new Error("Incomplete framebuffer: FRAMEBUFFER_INCOMPLETE_ATTACHMENT"); case e.FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT: throw new Error("Incomplete framebuffer: FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT"); case e.FRAMEBUFFER_INCOMPLETE_DIMENSIONS: throw new Error("Incomplete framebuffer: FRAMEBUFFER_INCOMPLETE_DIMENSIONS"); case e.FRAMEBUFFER_UNSUPPORTED: throw new Error("Incomplete framebuffer: FRAMEBUFFER_UNSUPPORTED"); default: throw new Error(`Incomplete framebuffer: ${i}`); } return this; } // @todo break this out more? setSize(e, a) { const t = this.gl; t.bindTexture(t.TEXTURE_2D, this.texture); let r; try { this.format === t.FLOAT ? (r = new Float32Array(e * a * 4), t.texImage2D(t.TEXTURE_2D, 0, t.RGBA, e, a, 0, t.DEPTH_COMPONENT, t.FLOAT, r)) : t.texImage2D(t.TEXTURE_2D, 0, t.RGBA, e, a, 0, t.RGBA, this.format, null); } catch { this.format === t.UNSIGNED_SHORT_4_4_4_4 ? (r = new Uint16Array(e * a * 4), t.texImage2D(t.TEXTURE_2D, 0, t.RGBA, e, a, 0, t.RGBA, t.UNSIGNED_SHORT_4_4_4_4, r)) : (r = new Uint8Array(e * a * 4), t.texImage2D(t.TEXTURE_2D, 0, t.RGBA, e, a, 0, t.RGBA, t.UNSIGNED_BYTE, r)); } t.bindTexture(t.TEXTURE_2D, null), t.bindRenderbuffer(t.RENDERBUFFER, this.renderbuffer), t.renderbufferStorage(t.RENDERBUFFER, t.DEPTH_COMPONENT16, e, a), t.bindRenderbuffer(t.RENDERBUFFER, null); } unload() { this.gl.deleteFramebuffer(this.framebuffer), this.gl.deleteRenderbuffer(this.renderbuffer), this.gl.deleteTexture(this.texture); } } const x = { video: { ready: "readyState", load: "canplay", width: "videoWidth", height: "videoHeight" }, img: { ready: "complete", load: "load", width: "width", height: "height" }, canvas: { ready: "complete", load: "load", width: "width", height: "height" } }; class b { _gl = null; _keys = []; _media = null; _data = null; _initialized = !1; _mediaTexture = null; _paintShader = null; _alphaFramebuffer = null; _downsampleCanvas = null; _downsampleContext = null; _vertexPositionBuffer = null; _vertexIndexBuffer = null; _texCoordBuffer = null; _vertexArrayObject = null; _downsampleWidth = 16; _downsampleHeight = 16; _hasAutoKeys = !1; /** * Creates a new GLChromaKey instance * @param source Source video, image or canvas element to key * @param target Target canvas element on which to paint keyed image(s) */ constructor(e, a) { if (!this.supportsWebGL2()) throw new Error("Browser does not support WebGL 2"); this._keys = [], this.source(e), this.target(a), this.buildWebGlBuffers(), this._initialized = !0, this.checkReady(); } buildWebGlBuffers() { const e = this._gl, a = e.createBuffer(); e.bindBuffer(e.ARRAY_BUFFER, a); const t = new Float32Array([ -1, -1, 0, // bottom left 1, -1, 0, // bottom right 1, 1, 0, // top right -1, 1, 0 // top left ]); e.bufferData(e.ARRAY_BUFFER, t, e.STATIC_DRAW), a.itemSize = 3, a.numItems = 4; const r = e.createBuffer(); e.bindBuffer(e.ARRAY_BUFFER, r); const i = new Float32Array([ 0, 1, // bottom left 1, 1, // bottom right 1, 0, // top right 0, 0 // top left ]); e.bufferData(e.ARRAY_BUFFER, i, e.STATIC_DRAW), r.itemSize = 2, r.numItems = 4; const s = e.createBuffer(); e.bindBuffer(e.ELEMENT_ARRAY_BUFFER, s); const o = new Uint16Array([ 0, 2, 1, // first triangle 0, 3, 2 // second triangle ]); e.bufferData(e.ELEMENT_ARRAY_BUFFER, o, e.STATIC_DRAW), s.itemSize = 1, s.numItems = 6, this._vertexPositionBuffer = a, this._vertexIndexBuffer = s, this._texCoordBuffer = r, this._vertexArrayObject = e.createVertexArray(); } setUpShaders() { const e = this._gl; let a = ""; if (this._hasAutoKeys = !1, this._keys.forEach((t) => { const r = typeof t == "string" ? { color: t } : Array.isArray(t) ? { color: t } : t; r.color === "auto" && (this._hasAutoKeys = !0); const i = r.color === "auto" ? "auto()" : `vec3(${r.color[0] / 255}, ${r.color[1] / 255}, ${r.color[2] / 255})`, s = isNaN(r.tolerance) ? 0.1 : r.tolerance.toFixed(3), o = isNaN(r.smoothness) ? 0.1 : r.smoothness.toFixed(3), n = isNaN(r.spill) ? 0.1 : r.spill.toFixed(3); a += `pixel = ProcessChromaKey(vTexCoord, ${i}, ${s}, ${o}, ${n}); `, r.debug && (a += `debug(); `); }), this._paintShader && this._paintShader.unload(), this._alphaFramebuffer && this._alphaFramebuffer.unload(), this._downsampleCanvas = null, this._downsampleContext = null, this._paintShader = new p( e, E, _.replace("%keys%", a) ), this._hasAutoKeys ? (this.calculateDownsampleSize(), this._alphaFramebuffer = new T(e, this._downsampleWidth, this._downsampleHeight), this._downsampleCanvas = document.createElement("canvas"), this._downsampleCanvas.width = this._downsampleWidth, this._downsampleCanvas.height = this._downsampleHeight, this._downsampleContext = this._downsampleCanvas.getContext("2d")) : (this._alphaFramebuffer = null, this._downsampleCanvas = null, this._downsampleContext = null), this._vertexArrayObject && this._paintShader) { e.bindVertexArray(this._vertexArrayObject); const t = this._paintShader; e.enableVertexAttribArray(t.location_position), e.enableVertexAttribArray(t.location_texCoord), e.bindBuffer(e.ARRAY_BUFFER, this._texCoordBuffer), e.vertexAttribPointer( t.location_texCoord, this._texCoordBuffer.itemSize, e.FLOAT, !1, // no normalization 0, // stride (0 = tightly packed) 0 // offset ), e.bindBuffer(e.ARRAY_BUFFER, this._vertexPositionBuffer), e.vertexAttribPointer( t.location_position, this._vertexPositionBuffer.itemSize, e.FLOAT, !1, // no normalization 0, // stride (0 = tightly packed) 0 // offset ), e.bindBuffer(e.ELEMENT_ARRAY_BUFFER, this._vertexIndexBuffer), e.bindVertexArray(null); } } calculateDownsampleSize() { if (!this._media || !this._data) return; const e = this._media[this._data.width] || 16, a = this._media[this._data.height] || 16, t = e / a, r = 256; t >= 1 ? (this._downsampleWidth = Math.round(Math.sqrt(r * t)), this._downsampleHeight = Math.round(this._downsampleWidth / t)) : (this._downsampleHeight = Math.round(Math.sqrt(r / t)), this._downsampleWidth = Math.round(this._downsampleHeight * t)), this._downsampleWidth = Math.max(8, Math.min(64, this._downsampleWidth)), this._downsampleHeight = Math.max(8, Math.min(64, this._downsampleHeight)); } initializeTextures() { const e = this._gl, a = (t) => { const r = e.createTexture(); return r.image = t, e.bindTexture(e.TEXTURE_2D, r), e.texImage2D(e.TEXTURE_2D, 0, e.RGBA, e.RGBA, e.UNSIGNED_BYTE, r.image), e.texParameteri(e.TEXTURE_2D, e.TEXTURE_MAG_FILTER, e.LINEAR), e.texParameteri(e.TEXTURE_2D, e.TEXTURE_MIN_FILTER, e.LINEAR), e.texParameteri(e.TEXTURE_2D, e.TEXTURE_WRAP_S, e.CLAMP_TO_EDGE), e.texParameteri(e.TEXTURE_2D, e.TEXTURE_WRAP_T, e.CLAMP_TO_EDGE), e.bindTexture(e.TEXTURE_2D, null), r.height = r.image.height, r.width = r.image.width, r; }; this._mediaTexture = a(this._media); } drawScreen(e, a, t) { const r = this._gl; e.useProgram(), r.bindVertexArray(this._vertexArrayObject), a && (e.set_source(0), r.activeTexture(r.TEXTURE0), r.bindTexture(r.TEXTURE_2D, a)), t && (e.set_alpha && e.set_alpha(1), r.activeTexture(r.TEXTURE1), r.bindTexture(r.TEXTURE_2D, t)), r.blendFunc(r.SRC_ALPHA, r.ONE_MINUS_SRC_ALPHA), r.enable(r.BLEND), r.disable(r.DEPTH_TEST), r.drawElements( r.TRIANGLES, this._vertexIndexBuffer.numItems, r.UNSIGNED_SHORT, 0 ), r.bindVertexArray(null); } checkReady() { if (!this._initialized) return; !this._data.ready || !this._data.load || this._media[this._data.ready] === this._data.readyTarget || this._data.readyTarget === void 0 && this._media[this._data.ready] ? (this.initializeTextures(), this.setUpShaders(), this.render()) : setTimeout(() => { this.checkReady(); }, 0); } /** * Returns true if browser supports WebGL 2, else false. */ supportsWebGL2() { try { return !!(window.WebGLRenderingContext && document.createElement("canvas").getContext("webgl2")); } catch { return !1; } } /** * Sets a new source video, image or canvas element to key. */ source(e) { if (!e || !e.tagName) throw new Error("Missing source element"); const a = e.tagName.toLowerCase(); if (this._data = x[a], !this._data) throw new Error("Unsupported source media type"); return this._media = e, this.checkReady(), this; } /** * Sets a new target canvas on which to paint keyed image(s). The context webgl2 will be used. */ target(e) { if (e instanceof HTMLCanvasElement) this._gl = e.getContext("webgl2"); else if (e instanceof WebGL2RenderingContext) this._gl = e; else throw new Error("Target must be an HTMLCanvasElement (or its WebGL2RenderingContext)"); if (!this._gl) throw new Error("Failed to get WebGL2 context from canvas"); return this.setUpShaders(), this; } /** * Returns the coordinates of a bounding box around non-transparent pixels in the form [x1, y1, x2, y2] */ getContentBounds() { const e = this._gl, a = e.canvas, t = a.width, r = a.height, i = new Uint8Array(t * r * 4); e.readPixels(0, 0, t, r, e.RGBA, e.UNSIGNED_BYTE, i); let s = t, o = r, n = -1, h = -1; for (let l = 0; l < r; l++) for (let f = 0; f < t; f++) { const u = (l * t + f) * 4; i[u + 3] > 0 && (f < s && (s = f), f > n && (n = f), l < o && (o = l), l > h && (h = l)); } if (n === -1) return [0, 0, t - 1, r - 1]; const d = r - 1 - h, m = r - 1 - o; return [s, d, n, m]; } /** * Updates frame from source element and paints to target canvas * @param options Render options object */ render(e = {}) { const { passthrough: a = !1 } = e; if (!this._mediaTexture || !this._mediaTexture.image || !this._paintShader || !this._media[this._data.ready]) return this; const t = this._gl; return t.bindTexture(t.TEXTURE_2D, this._mediaTexture), t.texImage2D(t.TEXTURE_2D, 0, t.RGBA, t.RGBA, t.UNSIGNED_BYTE, this._mediaTexture.image), t.bindTexture(t.TEXTURE_2D, null), a ? (t.viewport(0, 0, t.canvas.width, t.canvas.height), t.bindFramebuffer(t.FRAMEBUFFER, null), t.clearColor(0, 0, 0, 0), t.clear(t.COLOR_BUFFER_BIT), this.drawScreen(this._paintShader, this._mediaTexture, null), this) : (this._hasAutoKeys && this._alphaFramebuffer && this._downsampleCanvas && this._downsampleContext && (this._downsampleContext.drawImage(this._media, 0, 0, this._downsampleWidth, this._downsampleHeight), t.bindTexture(t.TEXTURE_2D, this._alphaFramebuffer.texture), t.texImage2D(t.TEXTURE_2D, 0, t.RGBA, t.RGBA, t.UNSIGNED_BYTE, this._downsampleCanvas), t.bindTexture(t.TEXTURE_2D, null)), t.viewport(0, 0, t.canvas.width, t.canvas.height), t.bindFramebuffer(t.FRAMEBUFFER, null), this.drawScreen(this._paintShader, this._mediaTexture, this._alphaFramebuffer?.texture || null), this); } /** * Sets one or more key colors in RGB, replacing any prior settings. Calling without parameters * clears all key colors. The auto key color mode downsamples the source image, grabs each corner * pixel, and keys on the two pixels with the most similar color. It works best on videos or images * with simplistic backgrounds, and can cause flickering if the algorithm gets it wrong. Use with * caution. * * @param keys - One or more key configurations. Each key can be: * - `'auto'` - Automatic color detection * - `[r, g, b]` - RGB color array (0-255 range) * - Object with properties: * - `color: [r, g, b] | 'auto'` - Color to key * - `tolerance?: number` - Color tolerance (0-1, default: 0.1) * - `smoothness?: number` - Edge smoothness (0-1, default: 0.1) * - `spill?: number` - Spill suppression (0-1, default: 0.1) * - `debug?: boolean` - Enable debug visualization (default: false) */ key(...e) { return this._keys = [], e.length === 1 && Array.isArray(e[0]) && Array.isArray(e[0][0]) && (e = e[0]), e.forEach((a) => { if (Array.isArray(a) && typeof a[0] == "number") { if (a.length !== 3) throw new Error("Key color must be 'auto' or an array like [r, g, b]"); if (a.some((r) => isNaN(r))) throw new Error("Invalid key color component"); const t = { color: a }; this._keys.push(t); } else if (typeof a == "object" && a !== null && !Array.isArray(a)) { const t = a; if (Array.isArray(t.color) && t.color.length === 3) { if (t.color.some((r) => isNaN(r))) throw new Error("Invalid key color component"); } else if (t.color !== "auto") throw new Error("Key color must be 'auto' or an array like [r, g, b]"); if (t.tolerance !== void 0 && (isNaN(t.tolerance) || t.tolerance < 0)) throw new Error("Tolerance must be a non-negative number"); if (t.smoothness !== void 0 && (isNaN(t.smoothness) || t.smoothness < 0)) throw new Error("Smoothness must be a non-negative number"); if (t.spill !== void 0 && (isNaN(t.spill) || t.spill < 0)) throw new Error("Spill must be a non-negative number"); this._keys.push(t); } else if (a === "auto") this._keys.push({ color: "auto" }); else throw new Error("Unsupported chroma key type"); }), this.setUpShaders(), this.render(), this; } /** * Unload all shader and buffers */ unload() { return !this._gl || !this._paintShader ? this : (this._paintShader.unload(), this._alphaFramebuffer && this._alphaFramebuffer.unload(), this._downsampleCanvas = null, this._downsampleContext = null, this._gl.deleteBuffer(this._vertexPositionBuffer), this._gl.deleteBuffer(this._vertexIndexBuffer), this._gl.deleteBuffer(this._texCoordBuffer), this._vertexArrayObject && this._gl.deleteVertexArray(this._vertexArrayObject), this._gl.deleteTexture(this._mediaTexture), this); } } export { b as default };