@viji-dev/core/dist/artist-global.d.ts

Universal execution engine for Viji Creative scenes

declare namespace VijiCore {
  declare interface AnalysisConfiguration {
      fftSize?: number;
      smoothing?: number;
      frequencyBands?: FrequencyBand[];
      beatDetection?: boolean;
      onsetDetection?: boolean;
  }

  declare interface AudioAPI {
      isConnected: boolean;
      volume: {
          rms: number;
          peak: number;
      };
      beat?: {
          isKick: boolean;
          confidence: number;
      };
      bands: Record<string, number>;
      getFrequencyData: () => Uint8Array;
  }

  declare interface CaptureFrameOptions {
      /** MIME type for output, e.g., 'image/png', 'image/jpeg', 'image/webp' */
      type?: string;
      /**
       * Target resolution.
       * - number: scale factor relative to current canvas size (e.g., 0.5 = 50%)
       * - { width, height }: exact output size; if aspect ratio differs from canvas,
       *   the source is center-cropped to match the target aspect ratio before scaling
       */
      resolution?: number | {
          width: number;
          height: number;
      };
  }

  declare interface ColorConfig {
      label: string;
      description?: string;
      group?: string;
      category?: ParameterCategory;
  }

  declare interface ColorParameter {
      value: string;
      label: string;
      description?: string;
      group: string;
      category: ParameterCategory;
  }

  declare type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation';

  declare type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';

  declare interface FaceData {
      id: number;
      bounds: {
          x: number;
          y: number;
          width: number;
          height: number;
      };
      confidence: number;
      landmarks?: {
          x: number;
          y: number;
          z?: number;
      }[];
      expressions: {
          neutral: number;
          happy: number;
          sad: number;
          angry: number;
          surprised: number;
          disgusted: number;
          fearful: number;
      };
      headPose: {
          pitch: number;
          yaw: number;
          roll: number;
      };
  }

  declare type FrameRateMode = 'full' | 'half';

  declare interface FrequencyBand {
      name: string;
      min: number;
      max: number;
  }

  declare interface HandData {
      id: number;
      handedness: 'left' | 'right';
      confidence: number;
      bounds: {
          x: number;
          y: number;
          width: number;
          height: number;
      };
      landmarks: {
          x: number;
          y: number;
          z: number;
      }[];
      palm: {
          x: number;
          y: number;
          z: number;
      };
      fingers: {
          thumb: {
              tip: {
                  x: number;
                  y: number;
                  z: number;
              };
              extended: boolean;
          };
          index: {
              tip: {
                  x: number;
                  y: number;
                  z: number;
              };
              extended: boolean;
          };
          middle: {
              tip: {
                  x: number;
                  y: number;
                  z: number;
              };
              extended: boolean;
          };
          ring: {
              tip: {
                  x: number;
                  y: number;
                  z: number;
              };
              extended: boolean;
          };
          pinky: {
              tip: {
                  x: number;
                  y: number;
                  z: number;
              };
              extended: boolean;
          };
      };
      gestures: {
          fist: number;
          openPalm: number;
          peace: number;
          thumbsUp: number;
          pointing: number;
      };
  }

  declare interface ImageConfig {
      label: string;
      description?: string;
      group?: string;
      category?: ParameterCategory;
  }

  declare interface ImageParameter {
      value: ImageBitmap | OffscreenCanvas | null;
      label: string;
      description?: string;
      group: string;
      category: ParameterCategory;
  }

  declare interface KeyboardAPI {
      isPressed(key: string): boolean;
      wasPressed(key: string): boolean;
      wasReleased(key: string): boolean;
      activeKeys: Set<string>;
      pressedThisFrame: Set<string>;
      releasedThisFrame: Set<string>;
      lastKeyPressed: string;
      lastKeyReleased: string;
      shift: boolean;
      ctrl: boolean;
      alt: boolean;
      meta: boolean;
  }

  declare interface MouseAPI {
      x: number;
      y: number;
      isInCanvas: boolean;
      isPressed: boolean;
      leftButton: boolean;
      rightButton: boolean;
      middleButton: boolean;
      velocity: {
          x: number;
          y: number;
      };
      deltaX: number;
      deltaY: number;
      wheelDelta: number;
      wheelX: number;
      wheelY: number;
      wasPressed: boolean;
      wasReleased: boolean;
      wasMoved: boolean;
  }

  declare interface NumberConfig {
      min?: number;
      max?: number;
      step?: number;
      label: string;
      description?: string;
      group?: string;
      category?: ParameterCategory;
  }

  declare interface NumberParameter {
      value: number;
      min: number;
      max: number;
      step: number;
      label: string;
      description?: string;
      group: string;
      category: ParameterCategory;
  }

  declare type ParameterCategory = 'audio' | 'video' | 'interaction' | 'general';

  declare interface PoseData {
      confidence: number;
      landmarks: {
          x: number;
          y: number;
          z: number;
          visibility: number;
      }[];
      face: {
          x: number;
          y: number;
      }[];
      torso: {
          x: number;
          y: number;
      }[];
      leftArm: {
          x: number;
          y: number;
      }[];
      rightArm: {
          x: number;
          y: number;
      }[];
      leftLeg: {
          x: number;
          y: number;
      }[];
      rightLeg: {
          x: number;
          y: number;
      }[];
  }

  declare type Resolution = {
      width: number;
      height: number;
  };

  declare interface SegmentationData {
      mask: Uint8Array;
      width: number;
      height: number;
  }

  declare interface SelectConfig {
      options: string[] | number[];
      label: string;
      description?: string;
      group?: string;
      category?: ParameterCategory;
  }

  declare interface SelectParameter {
      value: string | number;
      options: string[] | number[];
      label: string;
      description?: string;
      group: string;
      category: ParameterCategory;
  }

  declare interface SliderConfig {
      min?: number;
      max?: number;
      step?: number;
      label: string;
      description?: string;
      group?: string;
      category?: ParameterCategory;
  }

  declare interface SliderParameter {
      value: number;
      min: number;
      max: number;
      step: number;
      label: string;
      description?: string;
      group: string;
      category: ParameterCategory;
  }

  declare interface TextConfig {
      label: string;
      description?: string;
      group?: string;
      category?: ParameterCategory;
      maxLength?: number;
  }

  declare interface TextParameter {
      value: string;
      maxLength?: number;
      label: string;
      description?: string;
      group: string;
      category: ParameterCategory;
  }

  declare interface ToggleConfig {
      label: string;
      description?: string;
      group?: string;
      category?: ParameterCategory;
  }

  declare interface ToggleParameter {
      value: boolean;
      label: string;
      description?: string;
      group: string;
      category: ParameterCategory;
  }

  declare interface TouchAPI {
      points: TouchPoint[];
      count: number;
      started: TouchPoint[];
      moved: TouchPoint[];
      ended: TouchPoint[];
      primary: TouchPoint | null;
      gestures: TouchGestureAPI;
  }

  declare interface TouchGestureAPI {
      isPinching: boolean;
      isRotating: boolean;
      isPanning: boolean;
      isTapping: boolean;
      pinchScale: number;
      pinchDelta: number;
      rotationAngle: number;
      rotationDelta: number;
      panDelta: {
          x: number;
          y: number;
      };
      tapCount: number;
      lastTapTime: number;
      tapPosition: {
          x: number;
          y: number;
      } | null;
  }

  declare interface TouchPoint {
      id: number;
      x: number;
      y: number;
      pressure: number;
      radius: number;
      radiusX: number;
      radiusY: number;
      rotationAngle: number;
      force: number;
      deltaX: number;
      deltaY: number;
      velocity: {
          x: number;
          y: number;
      };
      isNew: boolean;
      isActive: boolean;
      isEnding: boolean;
  }

  declare const VERSION = "0.2.20";

  declare interface VideoAPI {
      isConnected: boolean;
      currentFrame: OffscreenCanvas | null;
      frameWidth: number;
      frameHeight: number;
      frameRate: number;
      getFrameData: () => ImageData | null;
      faces: FaceData[];
      hands: HandData[];
      pose: PoseData | null;
      segmentation: SegmentationData | null;
      cv: {
          enableFaceDetection(enabled: boolean): Promise<void>;
          enableFaceMesh(enabled: boolean): Promise<void>;
          enableHandTracking(enabled: boolean): Promise<void>;
          enablePoseDetection(enabled: boolean): Promise<void>;
          enableBodySegmentation(enabled: boolean): Promise<void>;
          getActiveFeatures(): CVFeature[];
          isProcessing(): boolean;
      };
  }

  declare interface VijiAPI {
      canvas: OffscreenCanvas;
      ctx?: OffscreenCanvasRenderingContext2D;
      gl?: WebGL2RenderingContext;
      width: number;
      height: number;
      pixelRatio: number;
      time: number;
      deltaTime: number;
      frameCount: number;
      fps: number;
      audio: AudioAPI;
      video: VideoAPI;
      mouse: MouseAPI;
      keyboard: KeyboardAPI;
      touches: TouchAPI;
      slider: (defaultValue: number, config: SliderConfig) => SliderParameter;
      color: (defaultValue: string, config: ColorConfig) => ColorParameter;
      toggle: (defaultValue: boolean, config: ToggleConfig) => ToggleParameter;
      select: (defaultValue: string | number, config: SelectConfig) => SelectParameter;
      text: (defaultValue: string, config: TextConfig) => TextParameter;
      number: (defaultValue: number, config: NumberConfig) => NumberParameter;
      image: (defaultValue: null, config: ImageConfig) => ImageParameter;
      useContext(type: '2d'): OffscreenCanvasRenderingContext2D;
      useContext(type: 'webgl'): WebGLRenderingContext | WebGL2RenderingContext;
  }
}

declare const viji: VijiCore.VijiAPI;
declare function render(viji: VijiCore.VijiAPI): void;