@phantasy/live2d-sdk
Version:
Reusable SDK for Live2D models with TTS and lip sync integration
1,320 lines (1,309 loc) • 102 kB
JavaScript
import { useState, useRef, useEffect, useCallback } from 'react';
/**
* Simple event emitter for SDK internal communication
*/
class SDKEventEmitter {
constructor() {
this.events = new Map();
}
on(event, callback) {
if (!this.events.has(event)) {
this.events.set(event, new Set());
}
this.events.get(event).add(callback);
}
off(event, callback) {
const callbacks = this.events.get(event);
if (callbacks) {
callbacks.delete(callback);
if (callbacks.size === 0) {
this.events.delete(event);
}
}
}
emit(event, ...args) {
const callbacks = this.events.get(event);
if (callbacks) {
callbacks.forEach(callback => {
try {
callback(...args);
}
catch (error) {
console.error(`Error in event callback for "${String(event)}":`, error);
}
});
}
}
once(event, callback) {
const onceCallback = (...args) => {
callback(...args);
this.off(event, onceCallback);
};
this.on(event, onceCallback);
}
removeAllListeners() {
this.events.clear();
}
listenerCount(event) {
return this.events.get(event)?.size || 0;
}
}
/**
* Base TTS Provider Interface
* Defines the contract that all TTS providers must implement
*/
class BaseTTSProvider {
constructor(config) {
this.config = config;
}
destroy() {
// Default implementation - providers can override if needed
}
/**
* Clean text by removing action brackets and normalizing whitespace
*/
cleanText(text) {
return text
.replace(/\[.*?\]/g, '') // Remove action brackets
.replace(/\s+/g, ' ') // Normalize whitespace
.trim();
}
/**
* Validate that text is not empty after cleaning
*/
validateText(text) {
const cleaned = this.cleanText(text);
if (!cleaned) {
throw new Error('No text to synthesize after cleaning');
}
return cleaned;
}
}
/**
* Simple logger utility for the SDK
* Based on the Phantasy client app logger
*/
function createLogger(namespace) {
const prefix = `[Phantasy-SDK:${namespace}]`;
return {
debug: (...args) => {
if (process.env.NODE_ENV === "development") {
console.debug(prefix, ...args);
}
},
info: (...args) => {
console.info(prefix, ...args);
},
warn: (...args) => {
console.warn(prefix, ...args);
},
error: (...args) => {
console.error(prefix, ...args);
},
};
}
/**
* Alkahest TTS Provider
* Extracted and adapted from Phantasy AudioService
*/
class AlkahestProvider extends BaseTTSProvider {
constructor(config) {
super(config);
this.logger = createLogger('AlkahestProvider');
}
getProviderName() {
return 'alkahest';
}
async generateAudio(text) {
try {
this.logger.info('Generating audio with Alkahest for text:', text);
const cleanedText = this.validateText(text);
this.logger.debug('Cleaned text:', cleanedText);
const url = `${this.config.apiUrl}/audio/speech`;
const requestBody = {
model: this.config.model || 'tts-1',
input: cleanedText,
voice: this.config.voice || 'rally',
speed: this.config.speed || 1.0,
response_format: this.config.format || 'wav',
};
const headers = {
'Content-Type': 'application/json',
Accept: 'audio/wav',
};
// Add API key if provided
if (this.config.apiKey) {
headers['Authorization'] = `Bearer ${this.config.apiKey}`;
}
this.logger.debug('Alkahest request:', {
url,
headers: { ...headers, Authorization: headers.Authorization ? '[REDACTED]' : undefined },
body: requestBody,
hasApiKey: !!this.config.apiKey,
});
const response = await fetch(url, {
method: 'POST',
headers,
body: JSON.stringify(requestBody),
});
if (!response.ok) {
const errorText = await response.text().catch(() => 'No error details available');
throw new Error(`Alkahest API error (${response.status}): ${errorText}`);
}
const arrayBuffer = await response.arrayBuffer();
this.logger.info('Alkahest audio generated successfully');
return new Blob([arrayBuffer], { type: 'audio/wav' });
}
catch (error) {
this.logger.error('Error in Alkahest generation:', error);
throw error;
}
}
}
/**
* ElevenLabs TTS Provider
* Extracted and adapted from Phantasy AudioService
*/
class ElevenLabsProvider extends BaseTTSProvider {
constructor(config) {
super(config);
this.logger = createLogger('ElevenLabsProvider');
}
getProviderName() {
return 'elevenlabs';
}
async generateAudio(text) {
try {
this.logger.info('Generating audio with ElevenLabs for text:', text);
// Check if we have a valid API key
if (!this.config.apiKey || this.config.apiKey === 'your_api_key_here') {
throw new Error('ElevenLabs API key is missing or placeholder');
}
// Log masked API key for debugging
const maskedKey = this.config.apiKey.substring(0, 4) +
'...' +
this.config.apiKey.substring(this.config.apiKey.length - 4);
this.logger.debug(`Using ElevenLabs API key: ${maskedKey}`);
// Ensure we have a valid voice ID
if (!this.config.voiceId) {
throw new Error('Voice ID is required for ElevenLabs');
}
const cleanedText = this.validateText(text);
const url = `${this.config.apiUrl || 'https://api.elevenlabs.io/v1/text-to-speech'}/${this.config.voiceId}`;
this.logger.debug('Using ElevenLabs URL:', url);
const response = await fetch(url, {
method: 'POST',
headers: {
Accept: 'audio/mpeg',
'Content-Type': 'application/json',
'xi-api-key': this.config.apiKey,
},
body: JSON.stringify({
text: cleanedText,
model_id: this.config.model || 'eleven_monolingual_v1',
voice_settings: {
stability: this.config.stability || 0.5,
similarity_boost: this.config.similarityBoost || 0.5,
},
}),
});
if (!response.ok) {
const errorText = await response.text().catch(() => 'No error details available');
throw new Error(`ElevenLabs API error (${response.status}): ${errorText}`);
}
const arrayBuffer = await response.arrayBuffer();
this.logger.info('ElevenLabs audio generated successfully');
return new Blob([arrayBuffer], { type: 'audio/mpeg' });
}
catch (error) {
this.logger.error('Error in ElevenLabs generation:', error);
throw error;
}
}
}
/**
* GPT-SoVITS TTS Provider
* Extracted and adapted from Phantasy AudioService
*/
class GPTSoVITSProvider extends BaseTTSProvider {
constructor(config) {
super(config);
this.logger = createLogger('GPTSoVITSProvider');
}
getProviderName() {
return 'gptsovits';
}
async generateAudio(text) {
try {
this.logger.info('GPT-SoVITS generating audio for text:', text);
const cleanedText = this.validateText(text);
this.logger.debug('Cleaned text:', cleanedText);
const params = new URLSearchParams({
text: cleanedText,
text_lang: this.config.textLang || 'en',
ref_audio_path: this.config.refAudioPath || 'audio/premade/10000-rally.wav',
prompt_lang: this.config.promptLang || 'en',
prompt_text: this.config.promptText || 'Hello, I am Phantasy, your AI companion.',
text_split_method: this.config.textSplitMethod || 'cut5',
batch_size: (this.config.batchSize || 1).toString(),
media_type: this.config.mediaType || 'wav',
streaming_mode: (this.config.streamingMode || false).toString(),
speed: (this.config.speed || 1.0).toString(),
});
const url = `${this.config.apiUrl || 'http://localhost:9880'}?${params.toString()}`;
const response = await fetch(url, {
method: 'GET',
headers: {
Accept: 'audio/wav',
},
});
if (!response.ok) {
throw new Error(`GPT-SoVITS HTTP error! status: ${response.status}`);
}
const arrayBuffer = await response.arrayBuffer();
this.logger.info('GPT-SoVITS audio generated successfully');
return new Blob([arrayBuffer], { type: 'audio/wav' });
}
catch (error) {
this.logger.error('Error in GPT-SoVITS generation:', error);
throw error;
}
}
}
/**
* Audio Analyzer for Lip Sync
* Extracted and adapted from Phantasy AudioService
*/
const AUDIO_CONFIG = {
fftSize: 256,
smoothingTimeConstant: 0.8,
minDecibels: -90,
maxDecibels: -10,
lipSyncSensitivity: 0.8,
autoplayTimeout: 5000,
};
class AudioAnalyzer extends SDKEventEmitter {
constructor() {
super();
this.audioContext = null;
this.hasAttemptedAutoplay = false;
this.logger = createLogger("AudioAnalyzer");
}
async initialize() {
try {
this.logger.info("Initializing AudioAnalyzer...");
await this.initializeAudioContext();
this.setupUserGestureHandler();
this.logger.info("AudioAnalyzer initialized successfully");
}
catch (error) {
this.logger.warn("AudioAnalyzer initialization had issues, but will continue:", error);
this.setupUserGestureHandler();
}
}
async playWithAnalysis(audioBlob, callback) {
try {
this.logger.info("Playing audio with analysis");
// Create audio element
const audioElement = new Audio();
const objectUrl = URL.createObjectURL(audioBlob);
audioElement.src = objectUrl;
audioElement.volume = 0.8;
// Set up analysis if we have callback and context
if (callback && this.audioContext) {
await this.setupAudioAnalysis(audioElement, callback);
}
// Track autoplay attempts
const isFirstAttempt = !this.hasAttemptedAutoplay;
this.hasAttemptedAutoplay = true;
try {
// Ensure AudioContext is running
await this.initializeAudioContext();
// Play the audio
this.emit("playbackStarted");
await audioElement.play();
if (isFirstAttempt) {
this.logger.info("Autoplay successful");
}
}
catch (playError) {
this.logger.warn("Audio autoplay prevented:", playError);
// Simulate lip sync without actual sound if callback exists
if (callback) {
this.simulateLipSync(callback);
}
}
// Wait for audio to finish
await new Promise((resolve) => {
const cleanup = () => {
URL.revokeObjectURL(objectUrl);
this.emit("playbackEnded");
resolve();
};
audioElement.addEventListener("ended", cleanup);
// Fallback timeout
setTimeout(() => {
if (audioElement.paused) {
cleanup();
}
}, AUDIO_CONFIG.autoplayTimeout);
});
}
catch (error) {
const err = error instanceof Error ? error : new Error(String(error));
this.logger.error("Audio playback failed:", err);
this.emit("error", err);
throw err;
}
}
stop() {
this.logger.debug("Audio stop requested");
// Note: In a full implementation, we'd track current audio elements and stop them
}
destroy() {
try {
this.stop();
if (this.audioContext && this.audioContext.state !== "closed") {
this.audioContext.close();
this.audioContext = null;
}
this.removeAllListeners();
this.logger.info("AudioAnalyzer destroyed");
}
catch (error) {
this.logger.error("Error during AudioAnalyzer destruction:", error);
}
}
// Private methods
async initializeAudioContext() {
if (!this.audioContext) {
const AudioContextClass = window.AudioContext || window.webkitAudioContext;
if (!AudioContextClass) {
throw new Error("AudioContext not available in this browser");
}
this.audioContext = new AudioContextClass();
this.logger.info("AudioContext created successfully");
}
// Log current state for debugging
this.logger.info(`AudioContext state: ${this.audioContext.state}`);
// Handle suspended state (Chrome's autoplay policy)
if (this.audioContext.state === "suspended") {
this.logger.info("AudioContext is suspended - attempting to resume...");
try {
await Promise.race([
this.audioContext.resume(),
new Promise((_, reject) => setTimeout(() => reject(new Error('AudioContext resume timeout')), 100))
]);
this.logger.info("AudioContext resumed successfully");
}
catch (error) {
this.logger.warn("AudioContext suspended - will resume after user gesture");
}
}
}
setupUserGestureHandler() {
const handleUserGesture = async () => {
if (this.audioContext && this.audioContext.state === "suspended") {
try {
await this.audioContext.resume();
this.logger.info("AudioContext resumed after user gesture");
// Remove listeners after successful resume
document.removeEventListener("click", handleUserGesture);
document.removeEventListener("keydown", handleUserGesture);
document.removeEventListener("touchstart", handleUserGesture);
}
catch (error) {
this.logger.error("Failed to resume AudioContext on user gesture:", error);
}
}
};
// Listen for user gestures
document.addEventListener("click", handleUserGesture, { once: false });
document.addEventListener("keydown", handleUserGesture, { once: false });
document.addEventListener("touchstart", handleUserGesture, { once: false });
}
async setupAudioAnalysis(audioElement, callback) {
if (!this.audioContext)
return;
try {
const source = this.audioContext.createMediaElementSource(audioElement);
const analyser = this.audioContext.createAnalyser();
// Configure analyser for lip sync
analyser.fftSize = AUDIO_CONFIG.fftSize;
analyser.smoothingTimeConstant = AUDIO_CONFIG.smoothingTimeConstant;
analyser.minDecibels = AUDIO_CONFIG.minDecibels;
analyser.maxDecibels = AUDIO_CONFIG.maxDecibels;
// Connect audio graph
source.connect(analyser);
analyser.connect(this.audioContext.destination);
const dataArray = new Uint8Array(analyser.frequencyBinCount);
const updateLipSync = () => {
analyser.getByteFrequencyData(dataArray);
// Calculate volume for lip sync
let sum = 0;
for (let i = 0; i < dataArray.length; i++) {
sum += dataArray[i];
}
const average = (sum / dataArray.length / 255.0) * AUDIO_CONFIG.lipSyncSensitivity;
const finalVolume = Math.min(average, 1.0);
// Emit volume data event
this.emit("volumeData", finalVolume);
// Call callback
callback(finalVolume);
// Continue animation
if (!audioElement.paused) {
requestAnimationFrame(updateLipSync);
}
else {
callback(0); // Reset mouth
}
};
// Start analysis when audio plays
audioElement.addEventListener("play", () => {
this.logger.debug("Audio started playing, beginning analysis");
updateLipSync();
});
}
catch (error) {
this.logger.error("Error setting up audio analysis:", error);
this.emit("error", error instanceof Error ? error : new Error(String(error)));
}
}
simulateLipSync(callback) {
this.logger.info("Simulating lip sync without audio");
const startTime = Date.now();
const duration = 4000; // 4 seconds
const simulate = () => {
const elapsed = Date.now() - startTime;
if (elapsed >= duration) {
callback(0); // Reset mouth
return;
}
// Create natural looking mouth movement
const time = elapsed / 1000;
const frequency = 2; // Hz
const value = (Math.sin(time * frequency * Math.PI) + 1) / 4; // 0-0.5 range
this.emit("volumeData", value);
callback(value);
requestAnimationFrame(simulate);
};
simulate();
}
}
/**
* TTS Manager - Extracted from Phantasy AudioService
* Handles multiple TTS providers with fallbacks and audio analysis
*/
class TTSManager extends SDKEventEmitter {
constructor(config) {
super();
this._isReady = false;
this.logger = createLogger("TTSManager");
this.config = config;
this.primaryProvider = this.createProvider(config.primary);
if (config.fallback) {
this.fallbackProvider = this.createProvider(config.fallback);
}
this.audioAnalyzer = new AudioAnalyzer();
this.setupAudioAnalyzer();
}
async initialize() {
try {
this.logger.info("Initializing TTS Manager...");
await this.audioAnalyzer.initialize();
this._isReady = true;
this.logger.info("TTS Manager initialized successfully");
}
catch (error) {
this.logger.warn("TTS Manager AudioAnalyzer had issues, but continuing initialization:", error);
this._isReady = true;
this.logger.info("TTS Manager initialized (AudioContext may need user gesture)");
}
}
/**
* Generate audio from text using configured provider
*/
async generateAudio(text) {
this.ensureReady();
try {
this.logger.info("Generating audio with primary provider:", this.config.primary.provider);
return await this.primaryProvider.generateAudio(text);
}
catch (primaryError) {
this.logger.warn("Primary provider failed:", primaryError);
if (this.fallbackProvider) {
try {
this.logger.info("Attempting fallback provider:", this.config.fallback?.provider);
return await this.fallbackProvider.generateAudio(text);
}
catch (fallbackError) {
this.logger.error("Both primary and fallback providers failed");
this.logger.error("Primary error:", primaryError);
this.logger.error("Fallback error:", fallbackError);
const primaryMessage = primaryError instanceof Error
? primaryError.message
: String(primaryError);
throw new Error(`TTS generation failed with both providers: ${primaryMessage}`);
}
}
else {
throw primaryError;
}
}
}
/**
* Play audio blob with optional analysis for lip sync
*/
async playAudio(audioBlob) {
this.ensureReady();
try {
await this.audioAnalyzer.playWithAnalysis(audioBlob, this.audioDataCallback);
}
catch (error) {
const err = error instanceof Error ? error : new Error(String(error));
this.logger.error("Audio playback failed:", err);
this.emit("error", err);
throw err;
}
}
/**
* Set audio data callback for lip sync
*/
setAudioDataCallback(callback) {
this.audioDataCallback = callback;
this.logger.debug("Audio data callback set");
}
/**
* Stop current audio playback
*/
stop() {
if (this.isReady()) {
this.audioAnalyzer.stop();
}
}
/**
* Update TTS configuration
*/
updateConfig(newConfig) {
this.logger.info("Updating TTS configuration");
// Check if providers need to be recreated
if (newConfig.primary.provider !== this.config.primary.provider) {
this.primaryProvider = this.createProvider(newConfig.primary);
}
if (newConfig.fallback) {
if (!this.config.fallback ||
newConfig.fallback.provider !== this.config.fallback.provider) {
this.fallbackProvider = this.createProvider(newConfig.fallback);
}
}
this.config = newConfig;
}
/**
* Check if TTS manager is ready
*/
isReady() {
return this._isReady;
}
/**
* Get current configuration
*/
getConfig() {
return { ...this.config };
}
/**
* Destroy and clean up resources
*/
destroy() {
this.logger.info("Destroying TTS Manager...");
try {
this.stop();
this.audioAnalyzer.destroy();
this.removeAllListeners();
this._isReady = false;
this.logger.info("TTS Manager destroyed");
}
catch (error) {
this.logger.error("Error during TTS Manager destruction:", error);
}
}
// Private methods
createProvider(config) {
switch (config.provider) {
case "alkahest":
return new AlkahestProvider(config);
case "elevenlabs":
return new ElevenLabsProvider(config);
case "gptsovits":
return new GPTSoVITSProvider(config);
default:
throw new Error(`Unknown TTS provider: ${config.provider}`);
}
}
setupAudioAnalyzer() {
this.audioAnalyzer.on("error", (error) => {
this.logger.error("Audio analyzer error:", error);
this.emit("error", error);
});
this.audioAnalyzer.on("playbackStarted", () => {
this.logger.debug("Audio playback started");
});
this.audioAnalyzer.on("playbackEnded", () => {
this.logger.debug("Audio playback ended");
// Reset lip sync to neutral
if (this.audioDataCallback) {
this.audioDataCallback(0);
}
});
}
ensureReady() {
if (!this._isReady) {
throw new Error("TTS Manager not initialized. Call initialize() first.");
}
}
}
/**
* Live2D Expression Manager
* Handles loading and applying expressions to Live2D models
*/
class ExpressionManager {
constructor() {
this.expressionCache = new Map();
this.logger = createLogger('ExpressionManager');
}
/**
* Set expression on the model
*/
async setExpression(model, expression, expressionMappings) {
if (!model?.internalModel?.coreModel) {
this.logger.warn('Cannot set expression: model not loaded');
return;
}
try {
this.logger.info(`Setting expression: ${expression}`);
this.logger.debug(`Expression mappings:`, expressionMappings);
// Determine what we're dealing with
let expressionUrl = null;
let useBuiltIn = false;
// Case 1: Expression is already a full URL - use it directly
if (this.isFullUrl(expression)) {
expressionUrl = expression;
this.logger.debug(`Expression is a full URL: ${expressionUrl}`);
}
// Case 2: Expression is a key in the mappings - get the URL
else if (expressionMappings && expression in expressionMappings) {
expressionUrl = expressionMappings[expression];
this.logger.debug(`Found expression mapping: "${expression}" -> "${expressionUrl}"`);
}
// Case 3: No mappings exist - try built-in method with just the expression name
else if (!expressionMappings || Object.keys(expressionMappings).length === 0) {
useBuiltIn = true;
this.logger.debug(`No mappings configured, will use built-in method for: ${expression}`);
}
// Case 4: Mappings exist but expression not found - this is an error
else {
throw new Error(`Expression "${expression}" not found in configured mappings. Available: ${Object.keys(expressionMappings).join(', ')}`);
}
// Now apply the expression
if (expressionUrl) {
// We have a URL (either direct or from mapping) - load it
// IMPORTANT: Do NOT use model.expression() for URLs!
this.logger.debug(`Loading expression from URL: ${expressionUrl}`);
await this.loadAndApplyExpression(model, expressionUrl);
this.logger.info(`Expression "${expression}" loaded from URL successfully`);
}
else if (useBuiltIn) {
// No URL, no mappings - try the built-in method with just the name
// This will only work if the model has built-in expressions
this.logger.debug(`Trying built-in expression: ${expression}`);
await this.tryBuiltInExpression(model, expression);
}
}
catch (error) {
this.logger.error(`Failed to set expression "${expression}":`, error);
throw error;
}
}
/**
* Load expression data from file and apply to model
*/
async loadAndApplyExpression(model, expressionPath) {
try {
// Check cache first
if (this.expressionCache.has(expressionPath)) {
const cachedData = this.expressionCache.get(expressionPath);
this.applyExpressionData(model, cachedData);
return;
}
this.logger.debug(`Loading expression from: ${expressionPath}`);
const response = await fetch(expressionPath);
if (!response.ok) {
throw new Error(`HTTP error ${response.status}`);
}
const expressionData = await response.json();
this.logger.debug('Received expression data:', expressionData);
// Cache the expression data
this.expressionCache.set(expressionPath, expressionData);
// Apply the expression
this.applyExpressionData(model, expressionData);
}
catch (error) {
this.logger.warn(`Failed to load expression from ${expressionPath}:`, error);
throw error;
}
}
/**
* Apply expression data directly to model parameters
*/
applyExpressionData(model, expressionData) {
if (!model?.internalModel?.coreModel)
return;
// Handle different parameter field names
const parameters = expressionData.Parameters || expressionData.parameters;
if (!parameters || !Array.isArray(parameters)) {
throw new Error('Expression data does not contain valid parameters array');
}
this.logger.debug(`Applying ${parameters.length} expression parameters`);
try {
parameters.forEach(param => {
const paramId = param.Id;
const paramValue = param.Value;
const paramBlend = param.Blend || 'Add';
const paramIndex = model.internalModel.coreModel.getParameterIndex(paramId);
if (paramIndex > -1) {
model.internalModel.coreModel.setParameterValueByIndex(paramIndex, paramValue);
this.logger.debug(`Applied parameter ${paramId} = ${paramValue} (${paramBlend})`);
}
else {
this.logger.warn(`Parameter not found in model: ${paramId}`);
}
});
}
catch (error) {
this.logger.error('Error applying expression parameters:', error);
throw error;
}
}
/**
* Try using the model's built-in expression method as fallback
*/
async tryBuiltInExpression(model, expression) {
// SAFETY CHECK: Never use this method with URLs!
if (this.isFullUrl(expression)) {
throw new Error(`tryBuiltInExpression called with URL: ${expression}. This is a bug!`);
}
if (typeof model.expression === 'function') {
try {
// Built-in expressions expect simple names like "happy" or "happy.exp3.json"
// NOT full paths or URLs
const expressionName = expression.endsWith('.exp3.json')
? expression
: `${expression}.exp3.json`;
this.logger.debug(`Trying built-in expression method with simple name: ${expressionName}`);
// This calls the Live2D library's built-in expression loader
// which will look for the expression relative to the model's location
await model.expression(expressionName);
this.logger.info(`Applied built-in expression: ${expressionName}`);
}
catch (error) {
this.logger.error(`Built-in expression method failed:`, error);
this.logger.error(`Attempted expression name: ${expression}`);
// Don't throw - built-in expressions are optional
}
}
else {
this.logger.warn(`Model does not support built-in expressions for "${expression}"`);
}
}
/**
* Preload expressions for better performance
*/
async preloadExpressions(expressionMappings) {
this.logger.info(`Preloading ${Object.keys(expressionMappings).length} expressions`);
const loadPromises = Object.entries(expressionMappings).map(async ([name, path]) => {
try {
const response = await fetch(path);
if (response.ok) {
const data = await response.json();
this.expressionCache.set(path, data);
this.logger.debug(`Preloaded expression: ${name}`);
}
}
catch (error) {
this.logger.warn(`Failed to preload expression ${name}:`, error);
}
});
await Promise.allSettled(loadPromises);
this.logger.info('Expression preloading completed');
}
/**
* Get list of available expressions from mappings
*/
getAvailableExpressions(expressionMappings) {
if (expressionMappings) {
return Object.keys(expressionMappings);
}
// Return common expression names if no mappings provided
return ['default', 'happy', 'sad', 'surprised', 'angry', 'thinking', 'excited', 'bored', 'shy'];
}
/**
* Clear expression cache
*/
clearCache() {
this.expressionCache.clear();
this.logger.debug('Expression cache cleared');
}
/**
* Get cache statistics
*/
getCacheStats() {
return {
size: this.expressionCache.size,
expressions: Array.from(this.expressionCache.keys()),
};
}
/**
* Check if a path is a full URL
*/
isFullUrl(path) {
return path.startsWith('http://') || path.startsWith('https://');
}
}
/**
* Live2D Parameter Mapper
* Handles parameter discovery and mapping for different model formats
*/
class ParameterMapper {
constructor() {
this.parameterCache = new Map();
this.logger = createLogger('ParameterMapper');
}
/**
* Find a parameter by trying multiple common names
*/
findParameter(model, parameterNames) {
if (!model?.internalModel?.coreModel)
return -1;
// Check cache first
const cacheKey = parameterNames.join('|');
if (this.parameterCache.has(cacheKey)) {
return this.parameterCache.get(cacheKey);
}
// Try to find parameter
for (const paramName of parameterNames) {
const paramIndex = model.internalModel.coreModel.getParameterIndex(paramName);
if (paramIndex > -1) {
this.logger.debug(`Found parameter: ${paramName} at index ${paramIndex}`);
this.parameterCache.set(cacheKey, paramIndex);
return paramIndex;
}
}
this.logger.warn(`No parameter found for names: ${parameterNames.join(', ')}`);
this.parameterCache.set(cacheKey, -1);
return -1;
}
/**
* Set parameter value if it exists
*/
setParameterIfExists(model, parameterNames, value) {
const paramIndex = this.findParameter(model, parameterNames);
if (paramIndex === -1)
return false;
try {
model.internalModel.coreModel.setParameterValueByIndex(paramIndex, value);
return true;
}
catch (error) {
this.logger.error(`Error setting parameter ${parameterNames[0]}:`, error);
return false;
}
}
/**
* Get parameter value if it exists
*/
getParameterIfExists(model, parameterNames) {
const paramIndex = this.findParameter(model, parameterNames);
if (paramIndex === -1)
return null;
try {
return model.internalModel.coreModel.getParameterValueByIndex(paramIndex);
}
catch (error) {
this.logger.error(`Error getting parameter ${parameterNames[0]}:`, error);
return null;
}
}
/**
* Discover and log all available parameters (useful for debugging)
*/
logAllParameters(model) {
if (!model?.internalModel?.coreModel) {
this.logger.warn('Cannot log parameters: model not loaded');
return [];
}
const parameters = [];
const paramCount = model.internalModel.coreModel.getParameterCount();
this.logger.info(`Model has ${paramCount} parameters:`);
for (let i = 0; i < paramCount; i++) {
const paramName = model.internalModel.coreModel.getParameterName(i);
const paramValue = model.internalModel.coreModel.getParameterValueByIndex(i);
parameters.push(paramName);
this.logger.info(` ${i}: ${paramName} = ${paramValue}`);
}
return parameters;
}
/**
* Get standard parameter mappings used across Phantasy implementations
*/
getStandardMappings() {
return {
// Mouth parameters for lip sync
mouth: [
'ParamMouthOpenY',
'PARAM_MOUTH_OPEN_Y',
'Param_Mouth_Open_Y',
'ParamMouthForm',
'PARAM_MOUTH_FORM',
'Param_Mouth_Form',
'ParamMouthOpen',
'PARAM_MOUTH_OPEN',
'Param_Mouth_Open',
],
// Eye parameters
leftEye: [
'ParamEyeLOpen',
'PARAM_EYE_L_OPEN',
'Param_Eye_L_Open',
],
rightEye: [
'ParamEyeROpen',
'PARAM_EYE_R_OPEN',
'Param_Eye_R_Open',
],
eyeBallX: [
'ParamEyeBallX',
'PARAM_EYE_BALL_X',
'Param_Eye_Ball_X',
],
eyeBallY: [
'ParamEyeBallY',
'PARAM_EYE_BALL_Y',
'Param_Eye_Ball_Y',
],
// Angle parameters
angleX: [
'ParamAngleX',
'PARAM_ANGLE_X',
'Param_Angle_X',
],
angleY: [
'ParamAngleY',
'PARAM_ANGLE_Y',
'Param_Angle_Y',
],
angleZ: [
'ParamAngleZ',
'PARAM_ANGLE_Z',
'Param_Angle_Z',
],
// Body parameters
bodyAngleX: [
'ParamBodyAngleX',
'PARAM_BODY_ANGLE_X',
'Param_Body_Angle_X',
],
bodyAngleY: [
'ParamBodyAngleY',
'PARAM_BODY_ANGLE_Y',
'Param_Body_Angle_Y',
],
bodyAngleZ: [
'ParamBodyAngleZ',
'PARAM_BODY_ANGLE_Z',
'Param_Body_Angle_Z',
],
// Breathing parameter
breath: [
'ParamBreath',
'PARAM_BREATH',
'Param_Breath',
],
};
}
/**
* Reset model to neutral position
*/
resetToNeutral(model) {
if (!model?.internalModel?.coreModel)
return;
const mappings = this.getStandardMappings();
// Reset all angle parameters to 0
this.setParameterIfExists(model, mappings.angleX, 0);
this.setParameterIfExists(model, mappings.angleY, 0);
this.setParameterIfExists(model, mappings.angleZ, 0);
this.setParameterIfExists(model, mappings.bodyAngleX, 0);
this.setParameterIfExists(model, mappings.bodyAngleY, 0);
this.setParameterIfExists(model, mappings.bodyAngleZ, 0);
// Reset eye position to center
this.setParameterIfExists(model, mappings.eyeBallX, 0);
this.setParameterIfExists(model, mappings.eyeBallY, 0);
// Reset mouth to closed
this.setParameterIfExists(model, mappings.mouth, 0);
this.logger.debug('Model reset to neutral position');
}
/**
* Clear parameter cache (useful when switching models)
*/
clearCache() {
this.parameterCache.clear();
this.logger.debug('Parameter cache cleared');
}
}
/**
* OrbitControls for Phantasy Live2D SDK
* Provides zoom and pan functionality for Live2D characters
*/
class OrbitControls {
constructor(_app, // Prefix with underscore to indicate it's intentionally unused
model, container, config = {}) {
this.logger = createLogger('OrbitControls');
this.isEnabled = true;
// State tracking
this.isDragging = false;
this.dragButton = -1;
this.lastPointerX = 0;
this.lastPointerY = 0;
// Transform state
this.currentZoom = 1;
this.panX = 0;
this.panY = 0;
this.originalX = 0;
this.originalY = 0;
// Touch state for mobile
this.touches = [];
this.lastPinchDistance = 0;
// Cleanup functions
this.cleanupFunctions = [];
console.log('OrbitControls: CONSTRUCTOR CALLED', {
config,
container: container?.tagName,
containerID: container?.id,
model: !!model,
});
// Note: app parameter kept for compatibility but not stored
this.model = model;
this.container = container;
this.config = this.getDefaultConfig(config);
console.log('OrbitControls: Final config:', this.config);
if (this.config.enabled) {
console.log('OrbitControls: Config enabled, calling initialize');
this.initialize();
}
else {
console.log('OrbitControls: Config disabled, NOT initializing');
}
}
getDefaultConfig(userConfig) {
return {
enabled: true,
enableZoom: true,
enablePan: true,
zoomSpeed: 1.0,
panSpeed: 1.0,
minZoom: 0.1,
maxZoom: 3.0,
zoomSensitivity: 0.01, // Increased for more responsive zooming
panBounds: {
left: -500,
right: 500,
top: -500,
bottom: 500,
},
mouseButtons: {
pan: 0, // Left mouse button
zoom: true, // Mouse wheel
},
touchGestures: {
pinchZoom: true,
panTouch: true,
},
...userConfig,
};
}
initialize() {
console.log('OrbitControls: INITIALIZING', {
enabled: this.config.enabled,
enableZoom: this.config.enableZoom,
enablePan: this.config.enablePan,
container: this.container,
model: !!this.model,
});
this.logger.info('Initializing orbit controls');
// Store original model position
if (this.model) {
this.originalX = this.model.x;
this.originalY = this.model.y;
this.logger.debug(`Stored original position: (${this.originalX}, ${this.originalY})`);
console.log('OrbitControls: Original position stored:', this.originalX, this.originalY);
}
// Set up mouse events
this.setupMouseEvents();
console.log('OrbitControls: Mouse events set up');
// Set up touch events for mobile
this.setupTouchEvents();
console.log('OrbitControls: Touch events set up');
// Set up wheel events for zoom
if (this.config.enableZoom && this.config.mouseButtons.zoom) {
this.setupWheelEvents();
console.log('OrbitControls: Wheel events set up');
}
// Apply initial transform
this.updateTransform();
console.log('OrbitControls: INITIALIZATION COMPLETE');
}
setupMouseEvents() {
const mouseDownHandler = (e) => {
console.log('OrbitControls: mousedown event', {
enabled: this.isEnabled,
enablePan: this.config.enablePan,
button: e.button,
expectedButton: this.config.mouseButtons.pan,
});
if (!this.isEnabled || !this.config.enablePan)
return;
if (e.button === this.config.mouseButtons.pan) {
this.isDragging = true;
this.dragButton = e.button;
this.lastPointerX = e.clientX;
this.lastPointerY = e.clientY;
this.container.style.cursor = 'grabbing';
e.preventDefault();
console.log('OrbitControls: drag started');
}
};
const mouseMoveHandler = (e) => {
if (!this.isEnabled || !this.isDragging || e.button !== -1)
return;
const deltaX = e.clientX - this.lastPointerX;
const deltaY = e.clientY - this.lastPointerY;
this.pan(deltaX, deltaY);
this.lastPointerX = e.clientX;
this.lastPointerY = e.clientY;
};
const mouseUpHandler = (e) => {
if (!this.isEnabled)
return;
if (e.button === this.dragButton) {
this.isDragging = false;
this.dragButton = -1;
this.container.style.cursor = 'default';
}
};
// Add event listeners
this.container.addEventListener('mousedown', mouseDownHandler);
document.addEventListener('mousemove', mouseMoveHandler);
document.addEventListener('mouseup', mouseUpHandler);
// Store cleanup functions
this.cleanupFunctions.push(() => {
this.container.removeEventListener('mousedown', mouseDownHandler);
document.removeEventListener('mousemove', mouseMoveHandler);
document.removeEventListener('mouseup', mouseUpHandler);
});
}
setupWheelEvents() {
const wheelHandler = (e) => {
console.log('OrbitControls: wheel event', {
enabled: this.isEnabled,
enableZoom: this.config.enableZoom,
deltaY: e.deltaY,
sensitivity: this.config.zoomSensitivity,
});
if (!this.isEnabled || !this.config.enableZoom)
return;
e.preventDefault();
// Determine zoom direction and amount
const delta = e.deltaY * this.config.zoomSensitivity;
const zoomFactor = 1 - delta;
console.log('OrbitControls: zooming', {
delta,
zoomFactor,
currentZoom: this.currentZoom,
});
this.zoom(zoomFactor, e.clientX, e.clientY);
};
this.container.addEventListener('wheel', wheelHandler, { passive: false });
this.cleanupFunctions.push(() => {
this.container.removeEventListener('wheel', wheelHandler);
});
}
setupTouchEvents() {
if (!this.config.touchGestures.pinchZoom && !this.config.touchGestures.panTouch)
return;
const touchStartHandler = (e) => {
if (!this.isEnabled)
return;
this.touches = Array.from(e.touches);
if (this.touches.length === 1 && this.config.touchGestures.panTouch) {
// Single touch - start panning
this.isDragging = true;
this.lastPointerX = this.touches[0].clientX;
this.lastPointerY = this.touches[0].clientY;
}
else if (this.touches.length === 2 && this.config.touchGestures.pinchZoom) {
// Two touches - start pinch zoom
this.isDragging = false;
this.lastPinchDistance = this.getPinchDistance();
}
e.preventDefault();
};
const touchMoveHandler = (e) => {
if (!this.isEnabled)
return;
this.touches = Array.from(e.touches);
if (this.touches.length === 1 && this.isDragging && this.config.touchGestures.panTouch) {
// Single touch panning
const deltaX = this.touches[0].clientX - this.lastPointerX;
const deltaY = this.touches[0].clientY - this.lastPointerY;
this.pan(deltaX, deltaY);
this.lastPointerX = this.touches[0].clientX;
this.lastPointerY = this.touches[0].clientY;
}
else if (this.touches.length === 2 && this.config.touchGestures.pinchZoom) {
// Pinch zoom
const pinchDistance = this.getPinchDistance();
const zoomFactor = pinchDistance / this.lastPinchDistance;
// Get center point of pinch
const centerX = (this.touches[0].clientX + this.touches[1].clientX) / 2;
const centerY = (this.touches[0].clientY + this.touches[1].clientY) / 2;
this.zoom(zoomFactor, centerX, centerY);
this.lastPinchDistance = pinchDistance;
}
e.preventDefault();
};
const touchEndHandler = (e) => {
if (!this.isEnabled)
return;
this.touches = Array.from(e.touches);
if (this.touches.length === 0) {
this.isDragging = false;
}
};
this.container.addEventListener('touchstart', touchStartHandler, {
passive: false,
});
this.container.addEventListener('touchmove', touchMoveHandler, {
passive: false,
});
this.container.addEventListener('touchend', touchEndHandler, {
passive: false,
});
this.cleanupFunctions.push(() => {
this.container.removeEventListener('touchstart', touchStartHandler);
this.container.removeEventListener('touchmove', touchMoveHandler);
this.container.removeEventListener('touchend', touchEndHandler);
});
}
getPinchDistance() {
if (this.touches.length < 2)
return 0;
const dx = this.touches[0].clientX - this.touches[1].clientX;
const dy = this.touches[0].clientY - this.touche