fix: fixes related to deck.gl upgrade (#3380)

* fix: polygon tool regression after deck.gl upgrade

Signed-off-by: Ihor Dykhta <ihordykhta@Ihors-MacBook-Pro.local>

* fix tile 3d layer crash; don't render polygon tool

Signed-off-by: Ihor Dykhta <ihordykhta@Ihors-MacBook-Pro.local>

* revert fragile injection

Signed-off-by: Ihor Dykhta <ihordykhta@Ihors-MacBook-Pro.local>

* fix stuck tiles

Signed-off-by: Ihor Dykhta <ihordykhta@Ihors-MacBook-Pro.local>

* fix polygon tool again

Signed-off-by: Ihor Dykhta <ihordykhta@Ihors-MacBook-Pro.local>

* pbr lighting to simple phong - eliminate radial specular/diffuse

Signed-off-by: Ihor Dykhta <ihordykhta@Ihors-MacBook-Pro.local>

* more fixes

Signed-off-by: Ihor Dykhta <ihordykhta@Ihors-MacBook-Pro.local>

* fix tests

Signed-off-by: Ihor Dykhta <ihordykhta@Ihors-MacBook-Pro.local>

* nit comments

Signed-off-by: Ihor Dykhta <ihordykhta@Ihors-MacBook-Pro.local>

---------

Signed-off-by: Ihor Dykhta <ihordykhta@Ihors-MacBook-Pro.local>
Co-authored-by: Ihor Dykhta <ihordykhta@Ihors-MacBook-Pro.local>

Author: igorDykhta

src/components/src/modals/export-video-modal.spec.tsx

@@ -158,6 +158,7 @@ describe('ExportVideoModal', () => {
       parameters: {blend: true},
       controller: true,
       views: {},
+      _isExport: true,
       effects: []
     });
   });

src/components/src/modals/export-video-modal.tsx

@@ -328,6 +328,7 @@ const ExportVideoModalFactory = () => {
     const deckPropsWithEffects = useMemo(
       () => ({
         ...hubbleDeckGlProps,
+        _isExport: true,
         effects: deckEffects
       }),
       [hubbleDeckGlProps, deckEffects]

src/components/src/plot-container.tsx

@@ -360,6 +360,7 @@ export default function PlotContainerFactory(
         isExport: true,
         deckGlProps: {
           ...mapFields.deckGlProps,
+          _isExport: true,
           useDevicePixels: false,
           deviceProps: {
             webgl: {

src/constants/src/default-settings.ts

@@ -1412,8 +1412,8 @@ export const LIGHT_AND_SHADOW_EFFECT: EffectDescription = {
   parameters: [
     {name: 'timestamp', min: 0, max: Number.MAX_SAFE_INTEGER},
     {name: 'shadowIntensity', min: 0, max: 1, defaultValue: DEFAULT_SHADOW_INTENSITY},
-    {name: 'sunLightIntensity', min: 0, max: 10, defaultValue: DEFAULT_LIGHT_INTENSITY},
-    {name: 'ambientLightIntensity', min: 0, max: 10, defaultValue: DEFAULT_LIGHT_INTENSITY},
+    {name: 'sunLightIntensity', min: 0, max: 5, defaultValue: DEFAULT_LIGHT_INTENSITY},
+    {name: 'ambientLightIntensity', min: 0, max: 5, defaultValue: DEFAULT_LIGHT_INTENSITY},
     {name: 'shadowColor', type: 'color', min: 0, max: 255, defaultValue: DEFAULT_SHADOW_COLOR},
     {name: 'sunLightColor', type: 'color', min: 0, max: 255, defaultValue: DEFAULT_LIGHT_COLOR},
     {name: 'ambientLightColor', type: 'color', min: 0, max: 255, defaultValue: DEFAULT_LIGHT_COLOR}

src/effects/src/custom-deck-lighting-effect.ts

@@ -4,8 +4,14 @@
 import {LightingEffect, shadow} from '@deck.gl/core';
 import type {Texture} from '@luma.gl/core';
 import type {ShaderModule} from '@luma.gl/shadertools';
+import {EDITOR_LAYER_ID} from '@kepler.gl/constants';
 import {patchTileViewportIds} from './tile-viewport-fix';
 
+// A plain LightingEffect() — its constructor calls _applyDefaultLights()
+// which populates the same default lights that deck.gl's EffectManager
+// uses internally. This is used to restore the original shading.
+const DEFAULT_LIGHTING_EFFECT = new LightingEffect();
+
 /**
  * Exposes private members of LightingEffect that we need to access.
  * These are runtime-accessible but TypeScript marks them as private.
@@ -20,6 +26,7 @@ interface LightingEffectPrivate {
 /** Extended shadow module props with our custom field. */
 interface CustomShadowProps {
   outputUniformShadow?: boolean;
+  useSimplePhong?: boolean;
   dummyShadowMap?: Texture | null;
   [key: string]: unknown;
 }
@@ -34,29 +41,78 @@ function insertBefore(source: string, target: string, textToInsert: string): str
 }
 
 /**
- * Create a patched shadow module that adds `outputUniformShadow` to the
- * shadow UBO. When true, `shadow_getShadowWeight` returns 1.0 (full
- * uniform shadow) instead of sampling the shadow map. Used for nighttime
- * rendering to avoid partial shadows from below.
+ * Create a patched shadow module that adds:
+ * - `outputUniformShadow`: when true, shadow_getShadowWeight returns 1.0
+ *   (full uniform shadow) instead of sampling the shadow map.
+ * - `useSimplePhong`: when true, replaces PBR lighting with simple
+ *   Lambertian diffuse (ambient + NdotL) for 3D tile layers, avoiding
+ *   the PBR microfacet specular artifacts on flat surfaces.
+ *
+ * Also fixes the vertex injection to be a no-op when shadows are disabled,
+ * preventing position corruption in billboard layers (e.g. the editor layer).
  */
 function createCustomShadowModule(): ShaderModule | null {
   if (!shadow) return null;
 
   const mod = {...shadow} as Record<string, any>;
 
-  const uboField = '  float outputUniformShadow;\n';
+  const uboField = '  float outputUniformShadow;\n  float useSimplePhong;\n';
   mod.vs = insertBefore(mod.vs, '} shadow;', uboField);
   mod.fs = insertBefore(mod.fs, '} shadow;', uboField);
 
+  // Add a varying to carry the common-space position for simple phong normals.
+  mod.vs = 'out vec3 custom_vWorldPos;\n' + mod.vs;
+  mod.fs = 'in vec3 custom_vWorldPos;\n' + mod.fs;
+
   mod.fs = insertBefore(
     mod.fs,
     'vec4 rgbaDepth = texture(shadowMap, position.xy);',
     'if (shadow.outputUniformShadow > 0.5) return 1.0;\n  '
   );
 
+  // Patch the VS so the fallback returns the caller's position instead of
+  // gl_Position (which is uninitialised when the hook fires in billboard
+  // layers like PathLayer / the editor polygon tool).
+  mod.vs = mod.vs
+    .replace(
+      'vec4 shadow_setVertexPosition(vec4 position_commonspace)',
+      'vec4 shadow_setVertexPosition(vec4 position_commonspace, vec4 currentPosition)'
+    )
+    .replace(
+      /return gl_Position;\s*\}/,
+      'return currentPosition;\n}'
+    );
+
+  mod.inject = {
+    ...shadow.inject,
+    'vs:DECKGL_FILTER_GL_POSITION': `
+    position = shadow_setVertexPosition(geometry.position, position);
+    custom_vWorldPos = geometry.position.xyz;
+    `,
+    'fs:DECKGL_FILTER_COLOR': `
+    #ifdef LIGHTING_FRAGMENT
+    if (shadow.useSimplePhong > 0.5) {
+      vec3 spDx = dFdx(custom_vWorldPos);
+      vec3 spDy = dFdy(custom_vWorldPos);
+      vec3 spN = normalize(cross(spDx, spDy));
+      vec3 spLit = lighting.ambientColor;
+      for (int i = 0; i < 3; i++) {
+        if (i >= lighting.directionalLightCount) break;
+        vec3 spDir = lighting_getDirectionalLight(i).direction;
+        float spNdotL = max(dot(spN, -normalize(spDir)), 0.0);
+        spLit += lighting_getDirectionalLight(i).color * spNdotL;
+      }
+      color.rgb *= spLit;
+    }
+    #endif
+    color = shadow_filterShadowColor(color);
+    `
+  };
+
   mod.uniformTypes = {
     ...shadow.uniformTypes,
-    outputUniformShadow: 'f32'
+    outputUniformShadow: 'f32',
+    useSimplePhong: 'f32'
   };
 
   const originalGetUniforms = shadow.getUniforms as (
@@ -66,6 +122,7 @@ function createCustomShadowModule(): ShaderModule | null {
   mod.getUniforms = (opts: CustomShadowProps = {}, context = {}) => {
     const u = originalGetUniforms(opts, context);
     u.outputUniformShadow = opts.outputUniformShadow ? 1.0 : 0.0;
+    u.useSimplePhong = opts.useSimplePhong ? 1.0 : 0.0;
     return u;
   };
 
@@ -74,12 +131,21 @@ function createCustomShadowModule(): ShaderModule | null {
 
 const CustomShadowModule = createCustomShadowModule();
 
+/**
+ * Detect layers that use the PBR shader module (3D tile sublayers:
+ * ScenegraphLayer sets `_lighting: 'pbr'`, SimpleMeshLayer sets `pbrMaterial`).
+ */
+function isPbrLayer(layer: any): boolean {
+  return layer.props?._lighting === 'pbr' || layer.props?.pbrMaterial !== undefined;
+}
+
 /**
  * Custom LightingEffect for kepler.gl.
  *
  * Extends deck.gl's LightingEffect with:
  * - A patched shadow module with `outputUniformShadow` for uniform shadow
  *   during nighttime (avoids partial shadows from below).
+ * - Simple phong replacement for PBR layers (avoids microfacet specular).
  * - getShaderModuleProps override that always provides dummyShadowMap
  *   to prevent "Bad texture binding" errors when shadows are disabled.
  */
@@ -110,6 +176,10 @@ class CustomDeckLightingEffect extends LightingEffect {
   preRender(opts) {
     if (!this._private.shadow) return;
 
+    // Filter editor layers out of the shadow pass so they don't cast shadows.
+    const originalLayers = opts.layers;
+    opts.layers = originalLayers.filter(l => !l.id.startsWith(EDITOR_LAYER_ID));
+
     let unpatch: (() => void) | undefined;
     if (this.isExportMode) {
       unpatch = patchTileViewportIds(opts);
@@ -118,6 +188,8 @@ class CustomDeckLightingEffect extends LightingEffect {
     super.preRender(opts);
 
     unpatch?.();
+
+    opts.layers = originalLayers;
   }
 
   cleanup(context) {
@@ -133,6 +205,34 @@ class CustomDeckLightingEffect extends LightingEffect {
   }
 
   getShaderModuleProps(layer, otherShaderModuleProps) {
+    // Skip lighting/shadow for the editor layer and its sublayers.
+    if (layer.id.startsWith(EDITOR_LAYER_ID)) {
+      return {
+        shadow: {
+          shadowEnabled: false,
+          dummyShadowMap: this._private.dummyShadowMap
+        },
+        lighting: {enabled: false},
+        phongMaterial: null,
+        gouraudMaterial: null
+      } as unknown as ReturnType<LightingEffect['getShaderModuleProps']>;
+    }
+
+    // When the effect is disabled (ghost kept alive to preserve the shadow
+    // shader module), delegate to a default LightingEffect so phong/gouraud
+    // layers get the same shading they had before the effect was ever added.
+    if (!this._private.shadow) {
+      const defaults = DEFAULT_LIGHTING_EFFECT.getShaderModuleProps(layer, otherShaderModuleProps);
+      return {
+        ...defaults,
+        shadow: {
+          shadowEnabled: false,
+          dummyShadowMap: this._private.dummyShadowMap
+        },
+        pbrMaterial: {unlit: 1}
+      } as unknown as ReturnType<LightingEffect['getShaderModuleProps']>;
+    }
+
     const props = super.getShaderModuleProps(layer, otherShaderModuleProps);
 
     if (
@@ -147,6 +247,21 @@ class CustomDeckLightingEffect extends LightingEffect {
       (props.shadow as CustomShadowProps).outputUniformShadow = this.outputUniformShadow;
     }
 
+    if (isPbrLayer(layer)) {
+      // PBR layers: disable PBR lighting (unlit=1 → outputs base texture color),
+      // keep directional lights in the lighting uniforms, and enable our simple
+      // phong replacement in the DECKGL_FILTER_COLOR hook. This gives us
+      // ambient + NdotL diffuse without PBR microfacet specular artifacts.
+      // Light direction is sky→surface (deck.gl convention), negated in GLSL.
+      (props as any).pbrMaterial = {unlit: 1};
+      if (props.shadow) {
+        (props.shadow as CustomShadowProps).useSimplePhong = true;
+      }
+      return props;
+    }
+
+    (props as any).pbrMaterial = {unlit: 0};
+
     return props;
   }
 }

src/layers/src/tile3d-layer/tile3d-layer.ts

@@ -55,6 +55,12 @@ function _checkLightingActive(context: any): boolean {
  */
 // @ts-expect-error Types have separate declarations of a private property '_loadTileset'.
 class KeplerTile3DLayer extends DeckTile3DLayer {
+  shouldUpdateState(params: any): boolean {
+    if (super.shouldUpdateState(params)) return true;
+    const lightingActive = _checkLightingActive(this.context);
+    return lightingActive !== ((this.state as any)?._lightingWasActive ?? false);
+  }
+
   // deck.gl Tile3DLayer.updateState only sets `needsUpdate` on cached tile
   // sub-layers when `propsChanged` fires, but NOT when `extensionsChanged`
   // fires (e.g. the shadow module being added/removed). The LayerManager sets
@@ -83,10 +89,7 @@ class KeplerTile3DLayer extends DeckTile3DLayer {
       const {layerMap} = this.state as any;
       if (layerMap) {
         for (const key in layerMap) {
-          const entry = layerMap[key];
-          if (entry?.layer) {
-            entry.layer.needsUpdate = true;
-          }
+          layerMap[key].needsUpdate = true;
         }
       }
     }
@@ -120,6 +123,13 @@ class KeplerTile3DLayer extends DeckTile3DLayer {
     const viewportsNumber = Object.keys(viewports).length;
     if (!timeline || !viewportsNumber || !tileset3d) return;
 
+    // We want higher detail for video/image export.
+    const isExporting = (this.context as any)?.deck?.props?._isExport;
+    if (isExporting) {
+      const baseSSE: number = tileset3d.options?.maximumScreenSpaceError ?? 8;
+      tileset3d.memoryAdjustedScreenSpaceError = baseSSE / 2;
+    }
+
     tileset3d
       .selectTiles(Object.values(viewports))
       .then((frameNumber: number) => {
@@ -275,20 +285,19 @@ class KeplerTile3DLayer extends DeckTile3DLayer {
   }
 
   /**
-   * During video export (preserveDrawingBuffer), report the layer as not loaded
-   * until every selected tile has a renderable sublayer.  Hubble.gl's
-   * DeckAdapter.onAfterRender checks layer.isLoaded on every top-level layer
-   * before capturing a frame, so returning false here makes it wait until the
-   * LOD transition is complete — no frame is captured with missing tiles.
+   * During video/image export, report the layer as not loaded until every
+   * selected tile has a renderable sublayer.  Hubble.gl's DeckAdapter and
+   * PlotContainer check layer.isLoaded before capturing a frame, so
+   * returning false here makes them wait until the LOD transition is
+   * complete — no frame is captured with missing tiles.
    */
   get isLoaded(): boolean {
     const baseLoaded = super.isLoaded;
     if (!baseLoaded) {
       return false;
     }
 
-    const gl = this.context?.gl;
-    const isExporting = gl?.getContextAttributes?.()?.preserveDrawingBuffer;
+    const isExporting = (this.context as any)?.deck?.props?._isExport;
     if (!isExporting) {
       return true;
     }