shellquest/src/screens/game/drawing/layeredRenderer.ts

Terminal-based procedurally generated dungeon crawler

import {PixelCanvas} from './pixelCanvas.ts';
import {TileMap, TILE_SIZE} from '../TileMap.ts';
import {type OptimizedBuffer, RGBA} from '@opentui/core';
import type {Entity} from '../entities/Entity.ts';
import type {Camera} from '../Camera.ts';
import {type TileInput, type TileRotation, toTileRef} from '../level/types.ts';

/**
 * Entity interface for rendering
 */
export interface RenderableEntity {
  x: number;
  y: number;
  width: number;
  height: number;
  layer: number;
  render(canvas: PixelCanvas): void;
}

/**
 * Tile with position and optional rotation for rendering
 */
export interface RenderTile {
  tile: TileInput;
  x: number;
  y: number;
  /** Y position used for depth sorting (defaults to y). Set to bottom of sprite for proper occlusion. */
  sortY?: number;
}

/**
 * LayeredRenderer - Multi-layer tile and entity rendering using PixelCanvas
 *
 * Uses a single PixelCanvas internally and renders layers in order:
 * - Layer 0: Background tiles (ground/ceiling)
 * - Layer 1: Sprite layer (shadows, entities sorted by Y for depth)
 * - Layer 2: Top layer (wall crowns, overlays)
 * - Layer 3: UI/effects
 */
export class LayeredRenderer {
  private layers: PixelCanvas[] = [];

  constructor(
    private mainCanvas: PixelCanvas,
    private tileMap: TileMap,
    private camera: Camera,
    layerCount: number = 4,
    x: number = 0,
    y: number = 0,
  ) {
    const pixelWidth = camera.viewportWidth;
    const pixelHeight = camera.viewportHeight;

    // Create layers
    for (let i = 0; i < layerCount; i++) {
      this.layers.push(new PixelCanvas(pixelWidth, pixelHeight));
    }
  }

  /**
   * Render a tile to a specific layer with optional rotation
   */
  private renderTile(
    layerIndex: number,
    tile: TileInput,
    pixelX: number,
    pixelY: number,
    flipHorizontal: boolean = false,
  ): void {
    const tileRef = toTileRef(tile);
    const pixels = this.tileMap.getTilePixels(tileRef.name);

    if (!pixels || layerIndex >= this.layers.length) return;

    const rotation = tileRef.rotation ?? 0;

    this.renderPixels(layerIndex, pixelX, pixelY, flipHorizontal, pixels, rotation);
  }

  private renderPixels(
    layerIndex: number,
    pixelX: number,
    pixelY: number,
    flipHorizontal: boolean,
    pixels: RGBA[],
    rotation: TileRotation = 0,
  ): void {
    const layer = this.layers[layerIndex];
    pixelX = Math.floor(pixelX);
    pixelY = Math.floor(pixelY);

    for (let cy = 0; cy < TILE_SIZE; cy++) {
      for (let cx = 0; cx < TILE_SIZE; cx++) {
        // Apply rotation to source coordinates
        let srcX = cx;
        let srcY = cy;

        switch (rotation) {
          case 90:
            // 90° clockwise: dest(x,y) <- src(y, SIZE-1-x)
            srcX = cy;
            srcY = TILE_SIZE - 1 - cx;
            break;
          case 180:
            // 180°: dest(x,y) <- src(SIZE-1-x, SIZE-1-y)
            srcX = TILE_SIZE - 1 - cx;
            srcY = TILE_SIZE - 1 - cy;
            break;
          case 270:
            // 270° clockwise (90° counter-clockwise): dest(x,y) <- src(SIZE-1-y, x)
            srcX = TILE_SIZE - 1 - cy;
            srcY = cx;
            break;
        }

        // Apply horizontal flip after rotation
        if (flipHorizontal) {
          srcX = TILE_SIZE - 1 - srcX;
        }

        const pixelIndex = srcY * TILE_SIZE + srcX;
        const color = pixels[pixelIndex];
        if (color && color.a > 0) {
          const drawX = pixelX + cx;
          const drawY = pixelY + cy;
          layer.setPixel(drawX, drawY, color);
        }
      }
    }
  }

  /**
   * Clear all layers
   */
  clear(): void {
    for (let i = 0; i < this.layers.length; i++) {
      const layer = this.layers[i];
      const clearColor = i === 0 ? RGBA.fromHex('#0a0a0a') : RGBA.fromValues(0, 0, 0, 0);
      layer.clear(clearColor.r, clearColor.g, clearColor.b, clearColor.a);
    }
  }

  /**
   * Clear a specific layer
   */
  clearLayer(layerIndex: number): void {
    if (layerIndex >= this.layers.length) return;
    const layer = this.layers[layerIndex];
    const clearColor = layerIndex === 0 ? RGBA.fromHex('#0a0a0a') : RGBA.fromValues(0, 0, 0, 0);
    layer.clear(clearColor.r, clearColor.g, clearColor.b, clearColor.a);
  }

  /**
   * Render a grid of tiles to a layer (for bottom layer)
   * Each tile can be a string or TileRef with rotation
   */
  renderTilesGridToLayer(
    layerIndex: number,
    tiles: TileInput[][],
    cameraPixelX: number = 0,
    cameraPixelY: number = 0,
  ): void {
    if (layerIndex >= this.layers.length) return;

    this.clearLayer(layerIndex);

    const startTileX = Math.floor(cameraPixelX / TILE_SIZE);
    const startTileY = Math.floor(cameraPixelY / TILE_SIZE);
    const endTileX = Math.ceil((cameraPixelX + this.camera.viewportWidth) / TILE_SIZE);
    const endTileY = Math.ceil((cameraPixelY + this.camera.viewportHeight) / TILE_SIZE);

    for (let tileY = startTileY; tileY < endTileY; tileY++) {
      for (let tileX = startTileX; tileX < endTileX; tileX++) {
        if (tileY >= 0 && tileY < tiles.length && tileX >= 0 && tileX < tiles[tileY].length) {
          const tile = tiles[tileY][tileX];
          if (tile) {
            const pixelX = tileX * TILE_SIZE - cameraPixelX;
            const pixelY = tileY * TILE_SIZE - cameraPixelY;
            this.renderTile(layerIndex, tile, pixelX, pixelY);
          }
        }
      }
    }
  }

  /**
   * Render a list of positioned tiles to a layer (for top layers)
   */
  renderTilesToLayer(
    layerIndex: number,
    tiles: RenderTile[],
    cameraPixelX: number = 0,
    cameraPixelY: number = 0,
  ): void {
    if (layerIndex >= this.layers.length) return;

    this.clearLayer(layerIndex);
    const viewportPixelWidth = this.camera.viewportWidth;
    const viewportPixelHeight = this.camera.viewportHeight;

    for (const renderTile of tiles) {
      const screenX = renderTile.x - cameraPixelX;
      const screenY = renderTile.y - cameraPixelY;

      if (
        screenX > -TILE_SIZE &&
        screenX < viewportPixelWidth &&
        screenY > -TILE_SIZE &&
        screenY < viewportPixelHeight
      ) {
        this.renderTile(layerIndex, renderTile.tile, screenX, screenY);
      }
    }
  }

  /**
   * Render entities to their specified layers
   */
  renderEntities(entities: Entity[], cameraPixelX: number = 0, cameraPixelY: number = 0): void {
    const usedLayers = new Set(entities.map((e) => e.layer));
    for (const layerIndex of usedLayers) {
      if (layerIndex < this.layers.length) {
        this.clearLayer(layerIndex);
      }
    }

    const sortedEntities = [...entities].sort((a, b) => a.y - b.y);
    const viewportPixelWidth = this.camera.viewportWidth;
    const viewportPixelHeight = this.camera.viewportHeight;

    for (const entity of sortedEntities) {
      const screenX = entity.x - this.camera.x;
      const screenY = entity.y - this.camera.y;

      if (
        screenX > -entity.width &&
        screenX < viewportPixelWidth &&
        screenY > -entity.height &&
        screenY < viewportPixelHeight
      ) {
        this.layers[entity.layer].save();
        this.layers[entity.layer].translate(-cameraPixelX, -cameraPixelY);
        this.layers[entity.layer].translate(entity.x, entity.y);
        entity.render(this.layers[entity.layer]);
        this.layers[entity.layer].restore();
      }
    }
  }

  renderToLayers(
    buffer: OptimizedBuffer,
    x: number,
    y: number,
    postProcess?: (canvas: PixelCanvas) => void,
  ): void {
    for (const layer of this.layers) {
      this.mainCanvas.drawCanvas(layer);
    }
    if (postProcess) {
      postProcess(this.mainCanvas);
    }
    this.mainCanvas.render(buffer, x, y);
  }

  /**
   * Render with a post-processing callback (for lighting, etc.)
   */

  getLayerCount(): number {
    return this.layers.length;
  }

  /**
   * Render entities and tiles together, sorted by Y position for proper depth.
   * This enables entities to appear in front of or behind tiles based on their Y position.
   * @param layerIndex The layer to render to
   * @param entities Array of entities to render
   * @param tiles Array of tiles to render (should have sortY set for proper ordering)
   * @param cameraPixelX Camera X offset in pixels
   * @param cameraPixelY Camera Y offset in pixels
   */
  renderYSorted(
    layerIndex: number,
    entities: Entity[],
    tiles: RenderTile[],
    cameraPixelX: number = 0,
    cameraPixelY: number = 0,
  ): void {
    if (layerIndex >= this.layers.length) return;

    this.clearLayer(layerIndex);
    const layer = this.layers[layerIndex];
    const viewportPixelWidth = this.camera.viewportWidth;
    const viewportPixelHeight = this.camera.viewportHeight;

    // Create sortable items for both entities and tiles
    type SortableItem =
      | {type: 'entity'; entity: Entity; sortY: number}
      | {type: 'tile'; tile: RenderTile; sortY: number};

    const items: SortableItem[] = [];

    // Add entities
    for (const entity of entities) {
      const screenX = entity.x - cameraPixelX;
      const screenY = entity.y - cameraPixelY;

      // Frustum culling
      if (
        screenX > -entity.width &&
        screenX < viewportPixelWidth &&
        screenY > -entity.height &&
        screenY < viewportPixelHeight
      ) {
        // Sort by the bottom of the entity (y + height) for proper depth
        items.push({
          type: 'entity',
          entity,
          sortY: entity.y + entity.height,
        });
      }
    }

    // Add tiles
    for (const renderTile of tiles) {
      const screenX = renderTile.x - cameraPixelX;
      const screenY = renderTile.y - cameraPixelY;

      // Frustum culling
      if (
        screenX > -TILE_SIZE &&
        screenX < viewportPixelWidth &&
        screenY > -TILE_SIZE &&
        screenY < viewportPixelHeight
      ) {
        // Use sortY if provided, otherwise use y + TILE_SIZE (bottom of tile)
        const sortY = renderTile.sortY ?? renderTile.y + TILE_SIZE;
        items.push({
          type: 'tile',
          tile: renderTile,
          sortY,
        });
      }
    }

    // Sort by Y position (lower Y renders first / behind)
    items.sort((a, b) => a.sortY - b.sortY);

    // Render in sorted order
    for (const item of items) {
      if (item.type === 'entity') {
        const entity = item.entity;
        layer.save();
        layer.translate(-cameraPixelX, -cameraPixelY);
        layer.translate(entity.x, entity.y);
        entity.render(layer);
        layer.restore();
      } else {
        const renderTile = item.tile;
        const screenX = renderTile.x - cameraPixelX;
        const screenY = renderTile.y - cameraPixelY;
        this.renderTile(layerIndex, renderTile.tile, screenX, screenY);
      }
    }
  }

  resize(width: number, height: number): void {
    for (const layer of this.layers) {
      layer.resize(width, height);
    }
  }
}