ImageOverlayPlugin: compose textures with the canvas API (#1415)

* Use canvas for texture composition

* Cleanup

* Remove use of "renderer"

* Remove texture cleanup

* Fix gaps

* Remove premultipliedAlpha assumption

* Fix up canvas composition

* Simplify

* Fix the flipping

* Cleanup

* Remove flipY

Author: gkjohnson

src/three/plugins/images/ImageOverlayPlugin.js

@@ -1,4 +1,4 @@
-import { WebGLRenderTarget, Color, SRGBColorSpace, BufferAttribute, Matrix4, Vector3, Box3, Triangle, CanvasTexture } from 'three';
+import { Color, SRGBColorSpace, BufferAttribute, Matrix4, Vector3, Box3, Triangle, CanvasTexture } from 'three';
 import { PriorityQueue, PriorityQueueItemRemovedError } from '3d-tiles-renderer/core';
 import { CesiumIonAuth, GoogleCloudAuth } from '3d-tiles-renderer/core/plugins';
 import { TiledTextureComposer } from './overlays/TiledTextureComposer.js';
@@ -126,7 +126,6 @@ export class ImageOverlayPlugin {
 		const {
 			overlays = [],
 			resolution = 256,
-			renderer = null,
 			enableTileSplitting = true,
 		} = options;
 
@@ -136,7 +135,6 @@ export class ImageOverlayPlugin {
 		this.priority = - 15;
 
 		// options
-		this.renderer = renderer;
 		this.resolution = resolution;
 		this._enableTileSplitting = enableTileSplitting;
 		this.overlays = [];
@@ -147,7 +145,6 @@ export class ImageOverlayPlugin {
 		this.tileComposer = null;
 		this.tileControllers = new Map();
 		this.overlayInfo = new Map();
-		this.usedTextures = new Set();
 		this.meshParams = new WeakMap();
 		this.pendingTiles = new Map();
 		this.processedTiles = new Set();
@@ -169,13 +166,7 @@ export class ImageOverlayPlugin {
 	// plugin functions
 	init( tiles ) {
 
-		if ( ! this.renderer ) {
-
-			throw new Error( 'ImageOverlayPlugin: "renderer" instance must be provided.' );
-
-		}
-
-		const tileComposer = new TiledTextureComposer( this.renderer );
+		const tileComposer = new TiledTextureComposer();
 		const processQueue = new PriorityQueue();
 		processQueue.maxJobs = 10;
 		processQueue.priorityCallback = ( a, b ) => {
@@ -380,7 +371,7 @@ export class ImageOverlayPlugin {
 
 				const { target } = tileInfo.get( tile );
 				bytes = bytes || 0;
-				bytes += MemoryUtils.getTextureByteLength( target?.texture );
+				bytes += MemoryUtils.getTextureByteLength( target );
 
 			}
 
@@ -1163,7 +1154,7 @@ export class ImageOverlayPlugin {
 
 		}
 
-		const { tiles, overlayInfo, resolution, tileComposer, tileControllers, usedTextures, processQueue } = this;
+		const { tiles, overlayInfo, resolution, tileComposer, tileControllers, processQueue } = this;
 		const { ellipsoid } = tiles;
 		const { controller, tileInfo } = overlayInfo.get( overlay );
 		const tileController = tileControllers.get( tile );
@@ -1241,12 +1232,13 @@ export class ImageOverlayPlugin {
 		let target = null;
 		if ( heightInRange && countTilesInRange( range, info.level, overlay ) !== 0 ) {
 
-			target = new WebGLRenderTarget( resolution, resolution, {
-				depthBuffer: false,
-				stencilBuffer: false,
-				generateMipmaps: false,
-				colorSpace: SRGBColorSpace,
-			} );
+			const canvas = document.createElement( 'canvas' );
+			canvas.width = resolution;
+			canvas.height = resolution;
+
+			target = new CanvasTexture( canvas );
+			target.colorSpace = SRGBColorSpace;
+			target.generateMipmaps = false;
 
 		}
 
@@ -1274,7 +1266,7 @@ export class ImageOverlayPlugin {
 						// if the previous layer is present then draw it as an overlay to fill in any gaps while we wait for
 						// the next set of textures
 						tileComposer.setRenderTarget( target, range );
-						tileComposer.clear( 0xffffff, 0 );
+						tileComposer.clear();
 
 						forEachTileInBounds( range, info.level - 1, tiling, ( tx, ty, tl ) => {
 
@@ -1284,8 +1276,6 @@ export class ImageOverlayPlugin {
 							if ( tex && ! ( tex instanceof Promise ) ) {
 
 								tileComposer.draw( tex, span );
-								usedTextures.add( tex );
-								this._scheduleCleanup();
 
 							}
 
@@ -1313,16 +1303,14 @@ export class ImageOverlayPlugin {
 
 					// draw the textures
 					tileComposer.setRenderTarget( target, range );
-					tileComposer.clear( 0xffffff, 0 );
+					tileComposer.clear();
 
 					forEachTileInBounds( range, info.level, tiling, ( tx, ty, tl ) => {
 
 						// draw using normalized bounds since the mercator bounds are non-linear
 						const span = tiling.getTileBounds( tx, ty, tl, true, false );
 						const tex = imageSource.get( tx, ty, tl );
 						tileComposer.draw( tex, span );
-						usedTextures.add( tex );
-						this._scheduleCleanup();
 
 					} );
 
@@ -1380,7 +1368,7 @@ export class ImageOverlayPlugin {
 				params.layerInfo.length = overlays.length;
 
 				// assign the uniforms
-				params.layerMaps.value[ i ] = target !== null ? target.texture : null;
+				params.layerMaps.value[ i ] = target !== null ? target : null;
 				params.layerInfo.value[ i ] = overlay;
 
 				// mark per-layer defines
@@ -1397,30 +1385,6 @@ export class ImageOverlayPlugin {
 
 	}
 
-	_scheduleCleanup() {
-
-		// clean up textures used for drawing the tile overlays
-		if ( ! this._cleanupScheduled ) {
-
-			this._cleanupScheduled = true;
-			requestAnimationFrame( () => {
-
-				const { usedTextures } = this;
-				usedTextures.forEach( tex => {
-
-					tex.dispose();
-
-				} );
-
-				usedTextures.clear();
-				this._cleanupScheduled = false;
-
-			} );
-
-		}
-
-	}
-
 	_markNeedsUpdate() {
 
 		if ( this.needsUpdate === false ) {

src/three/plugins/images/overlays/TiledTextureComposer.js

@@ -1,161 +1,59 @@
-import { ShaderMaterial, MathUtils, Vector2, PlaneGeometry, OrthographicCamera, Mesh, Color, DoubleSide } from 'three';
-
-const _camera = /* @__PURE__ */ new OrthographicCamera();
-const _color = /* @__PURE__ */ new Color();
+import { MathUtils } from 'three';
 
 // Utility for composing a series of tiled textures together onto a target texture in a given range
 export class TiledTextureComposer {
 
-	constructor( renderer ) {
+	constructor() {
 
-		this.renderer = renderer;
-		this.renderTarget = null;
+		this.canvas = null;
+		this.context = null;
 		this.range = [ 0, 0, 1, 1 ];
-		this.quad = new Mesh( new PlaneGeometry(), new ComposeTextureMaterial() );
 
 	}
 
 	// set the target render texture and the range that represents the full span
 	setRenderTarget( renderTarget, range ) {
 
-		this.renderTarget = renderTarget;
+		this.canvas = renderTarget.image;
+		this.context = renderTarget.image.getContext( '2d' );
 		this.range = [ ...range ];
 
 	}
 
 	// draw the given texture at the given span with the provided projection
 	draw( texture, span ) {
 
-		// draw the texture at the given sub range
-		const { range, renderer, quad, renderTarget } = this;
-		const material = quad.material;
-		material.map = texture;
-
-		// map the range to draw the texture to
-		material.minRange.x = MathUtils.mapLinear( span[ 0 ], range[ 0 ], range[ 2 ], - 1, 1 );
-		material.minRange.y = MathUtils.mapLinear( span[ 1 ], range[ 1 ], range[ 3 ], - 1, 1 );
-
-		material.maxRange.x = MathUtils.mapLinear( span[ 2 ], range[ 0 ], range[ 2 ], - 1, 1 );
-		material.maxRange.y = MathUtils.mapLinear( span[ 3 ], range[ 1 ], range[ 3 ], - 1, 1 );
-
-		// draw the texture
-		const currentRenderTarget = renderer.getRenderTarget();
-		const currentAutoClear = renderer.autoClear;
-		renderer.autoClear = false;
-		renderer.setRenderTarget( renderTarget );
-		renderer.render( quad, _camera );
-		renderer.setRenderTarget( currentRenderTarget );
-		renderer.autoClear = currentAutoClear;
-
-		material.map = null;
-
-	}
-
-	// clear the set target
-	clear( color, alpha = 1 ) {
-
-		// clear the texture
-		const { renderer, renderTarget } = this;
-		const currentRenderTarget = renderer.getRenderTarget();
-		const currentClearColor = renderer.getClearColor( _color );
-		const currentClearAlpha = renderer.getClearAlpha();
-
-		renderer.setClearColor( color, alpha );
-		renderer.setRenderTarget( renderTarget );
-		renderer.clear();
+		const { canvas, range, context } = this;
+		const { width, height } = canvas;
+		const { image } = texture;
+		const minX = Math.round( MathUtils.mapLinear( span[ 0 ], range[ 0 ], range[ 2 ], 0, width ) );
+		const minY = Math.round( MathUtils.mapLinear( span[ 1 ], range[ 1 ], range[ 3 ], 0, height ) );
+		const maxX = Math.round( MathUtils.mapLinear( span[ 2 ], range[ 0 ], range[ 2 ], 0, width ) );
+		const maxY = Math.round( MathUtils.mapLinear( span[ 3 ], range[ 1 ], range[ 3 ], 0, height ) );
 
-		renderer.setRenderTarget( currentRenderTarget );
-		renderer.setClearColor( currentClearColor, currentClearAlpha );
+		const imageWidth = maxX - minX;
+		const imageHeight = maxY - minY;
+		if ( ! ( image instanceof ImageBitmap ) ) {
 
-	}
-
-	dispose() {
-
-		this.quad.material.dispose();
-		this.quad.geometry.dispose();
-
-	}
-
-}
-
-// Draws the given texture with no depth testing at the given bounds defined by "minRange" and "maxRange"
-class ComposeTextureMaterial extends ShaderMaterial {
-
-	// the [ - 1, 1 ] NDC ranges to draw the texture at
-	get minRange() {
+			context.drawImage( image, minX, height - minY, imageWidth, - imageHeight );
 
-		return this.uniforms.minRange.value;
+		} else {
 
-	}
-
-	get maxRange() {
+			context.save();
+			context.translate( minX, height - minY );
+			context.scale( 1, - 1 );
+			context.drawImage( image, 0, 0, imageWidth, imageHeight );
+			context.restore();
 
-		return this.uniforms.maxRange.value;
+		}
 
 	}
 
-	// access the map being drawn
-	get map() {
-
-		return this.uniforms.map.value;
-
-	}
-
-	set map( v ) {
-
-		this.uniforms.map.value = v;
-
-	}
-
-	constructor() {
-
-		super( {
-			depthWrite: false,
-			depthTest: false,
-			transparent: false,
-			side: DoubleSide,
-			premultipliedAlpha: true,
-			uniforms: {
-				map: { value: null },
-
-				// the normalized [0, 1] range of the target to draw to
-				minRange: { value: new Vector2() },
-				maxRange: { value: new Vector2() },
-			},
-
-			vertexShader: /* glsl */`
-
-				uniform vec2 minRange;
-				uniform vec2 maxRange;
-				varying vec2 vUv;
-
-				void main() {
-
-					vUv = uv;
-					gl_Position = vec4( mix( minRange, maxRange, uv ), 0, 1 );
-
-				}
-
-			`,
-
-			fragmentShader: /* glsl */`
-
-				uniform sampler2D map;
-				uniform vec2 minRange;
-				uniform vec2 maxRange;
-				varying vec2 vUv;
-
-				void main() {
-
-					// sample the texture
-					gl_FragColor = texture( map, vUv );
-					#include <premultiplied_alpha_fragment>
-
-				}
-
-			`,
-		} );
+	// clear the set target
+	clear() {
 
+		const { context, canvas } = this;
+		context.clearRect( 0, 0, canvas.width, canvas.height );
 
 	}
 

src/three/plugins/images/overlays/wrapOverlaysMaterial.js

@@ -146,7 +146,7 @@ export function wrapOverlaysMaterial( material, previousOnBeforeCompile ) {
 
 								} else {
 
-									// premultiplied alpha equation
+									tint.rgb *= tint.a;
 									diffuseColor = tint + diffuseColor * ( 1.0 - tint.a );
 
 								}

src/three/plugins/images/sources/TiledImageSource.js

@@ -26,6 +26,7 @@ export class TiledImageSource extends DataCache {
 	// helper for processing the buffer into a texture
 	async processBufferToTexture( buffer ) {
 
+		// pre-flip the y axis
 		const blob = new Blob( [ buffer ] );
 		const imageBitmap = await createImageBitmap( blob, {
 			premultiplyAlpha: 'none',