@coze/uniapp-api
Version:
Official Coze UniApp SDK for seamless AI integration into your applications | 扣子官方 UniApp SDK，助您轻松集成 AI 能力到应用中
429 lines (428 loc) • 16.7 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.WsSpeechClient = void 0;
const api_1 = require("@coze/api");
const pcm_stream_player_1 = require("../pcm-stream-player");
const api_2 = require("../../api");
// Import types directly from @coze/api
/**
* WsSpeechClient for UniApp/WeChat Mini Program
* Handles text-to-speech streaming through WebSockets
* @class
*/
class WsSpeechClient {
/**
* Creates a new WsSpeechClient instance
* @param {WsToolsOptions} config - Configuration options
*/
constructor(config) {
Object.defineProperty(this, "ws", {
enumerable: true,
configurable: true,
writable: true,
value: null
});
Object.defineProperty(this, "listeners", {
enumerable: true,
configurable: true,
writable: true,
value: new Map()
});
Object.defineProperty(this, "pcmStreamPlayer", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "trackId", {
enumerable: true,
configurable: true,
writable: true,
value: 'default'
});
Object.defineProperty(this, "api", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "totalDuration", {
enumerable: true,
configurable: true,
writable: true,
value: 0
});
Object.defineProperty(this, "playbackStartTime", {
enumerable: true,
configurable: true,
writable: true,
value: null
});
Object.defineProperty(this, "playbackPauseTime", {
enumerable: true,
configurable: true,
writable: true,
value: null
});
Object.defineProperty(this, "playbackTimeout", {
enumerable: true,
configurable: true,
writable: true,
value: null
});
Object.defineProperty(this, "elapsedBeforePause", {
enumerable: true,
configurable: true,
writable: true,
value: 0
});
Object.defineProperty(this, "audioDeltaList", {
enumerable: true,
configurable: true,
writable: true,
value: []
});
Object.defineProperty(this, "sampleRate", {
enumerable: true,
configurable: true,
writable: true,
value: pcm_stream_player_1.PcmStreamPlayer.getSampleRate()
});
Object.defineProperty(this, "config", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
/**
* Process audio data from the queue
* @private
*/
Object.defineProperty(this, "handleAudioMessage", {
enumerable: true,
configurable: true,
writable: true,
value: () => __awaiter(this, void 0, void 0, function* () {
const message = this.audioDeltaList[0];
// Use UniApp's base64ToArrayBuffer instead of atob for Mini Program compatibility
const arrayBuffer = uni.base64ToArrayBuffer(message);
// Calculate duration in seconds
const bytesPerSecond = Number(this.sampleRate) * 1 * (16 / 8); // sampleRate * channels * (bitDepth/8)
const duration = arrayBuffer.byteLength / bytesPerSecond;
this.totalDuration += duration;
try {
yield this.pcmStreamPlayer.add16BitPCM(arrayBuffer, this.trackId);
// Start or update the playback timer
if (!this.playbackStartTime && !this.playbackPauseTime) {
this.playbackStartTime = Date.now();
this.elapsedBeforePause = 0;
}
// Remove the processed message and process the next one if available
this.audioDeltaList.shift();
if (this.audioDeltaList.length > 0) {
this.handleAudioMessage();
}
}
catch (error) {
console.warn(`[speech] pcmStreamPlayer error ${error === null || error === void 0 ? void 0 : error.message}`, error);
}
})
});
this.api = new api_2.CozeAPI(Object.assign({ baseWsURL: api_1.COZE_CN_BASE_WS_URL }, config));
this.pcmStreamPlayer = new pcm_stream_player_1.PcmStreamPlayer({
sampleRate: this.sampleRate,
});
this.config = config;
}
/**
* Initialize the WebSocket connection
* @returns {Promise<WebSocketAPI>} - The WebSocket API instance
*/
init() {
return __awaiter(this, void 0, void 0, function* () {
if (this.ws) {
return this.ws;
}
const ws = yield this.api.websockets.audio.speech.create(undefined, this.config.websocketOptions);
this.ws = ws;
let isResolved = false;
// Generate a unique track ID using timestamp instead of uuid for Mini Program
this.trackId = `my-track-id-${Date.now()}`;
this.totalDuration = 0;
if (this.playbackTimeout) {
clearTimeout(this.playbackTimeout);
this.playbackTimeout = null;
}
this.playbackStartTime = null;
return new Promise((resolve, reject) => {
ws.onopen = () => {
console.debug('[speech] ws open');
};
ws.onmessage = data => {
var _a, _b, _c, _d, _e;
// Trigger all registered event listeners
this.emit('data', data);
this.emit(data.event_type, data);
if (data.event_type === api_1.WebsocketsEventType.ERROR) {
this.closeWs();
if (isResolved) {
return;
}
isResolved = true;
// Handle error with type casting to access data properties safely
reject(new api_1.APIError((_a = data.data) === null || _a === void 0 ? void 0 : _a.code, {
code: (_b = data.data) === null || _b === void 0 ? void 0 : _b.code,
msg: (_c = data.data) === null || _c === void 0 ? void 0 : _c.msg,
detail: data.detail,
}, (_d = data.data) === null || _d === void 0 ? void 0 : _d.msg, undefined));
return;
}
else if (data.event_type === api_1.WebsocketsEventType.SPEECH_CREATED) {
resolve(ws);
isResolved = true;
}
else if (data.event_type === api_1.WebsocketsEventType.SPEECH_AUDIO_UPDATE) {
// Push audio data to queue for sequential processing
// Use type assertion to access data property safely
this.audioDeltaList.push((_e = data.data) === null || _e === void 0 ? void 0 : _e.delta);
if (this.audioDeltaList.length === 1) {
this.handleAudioMessage();
}
}
else if (data.event_type === api_1.WebsocketsEventType.SPEECH_AUDIO_COMPLETED) {
console.debug('[speech] totalDuration', this.totalDuration);
if (this.playbackStartTime) {
// Calculate remaining time = total duration - played time - paused time
const now = Date.now();
const remaining = this.totalDuration -
(now - this.playbackStartTime) / 1000 -
this.elapsedBeforePause;
this.playbackTimeout = setTimeout(() => {
this.emit('completed', undefined);
this.playbackStartTime = null;
this.elapsedBeforePause = 0;
}, remaining * 1000);
}
this.closeWs();
}
};
ws.onerror = (error, event) => {
var _a, _b;
console.error('[speech] WebSocket error', error, event);
this.emit('data', error);
this.emit(api_1.WebsocketsEventType.ERROR, error);
this.closeWs();
if (isResolved) {
return;
}
isResolved = true;
// Use type assertion to safely access error properties
reject(new api_1.APIError((_a = error.data) === null || _a === void 0 ? void 0 : _a.code, error, (_b = error.data) === null || _b === void 0 ? void 0 : _b.msg, undefined));
};
ws.onclose = () => {
console.debug('[speech] ws closed');
};
});
});
}
/**
* Connect to the speech service and configure audio output
* @param {Object} options - Connection options
* @param {string} [options.voiceId] - Voice ID to use
* @param {number} [options.speechRate] - Speech rate (-50 to 100, default 0)
* @returns {Promise<void>}
*/
connect() {
return __awaiter(this, arguments, void 0, function* ({ voiceId, speechRate, } = {}) {
var _a;
yield this.init();
(_a = this.ws) === null || _a === void 0 ? void 0 : _a.send({
id: `event-${Date.now()}`,
event_type: api_1.WebsocketsEventType.SPEECH_UPDATE,
data: {
output_audio: {
codec: 'pcm',
voice_id: voiceId || undefined,
speech_rate: speechRate || undefined,
pcm_config: {
sample_rate: this.sampleRate,
},
},
},
});
});
}
/**
* Disconnect from the speech service and stop audio playback
* @returns {Promise<void>}
*/
disconnect() {
return __awaiter(this, void 0, void 0, function* () {
if (this.playbackTimeout) {
clearTimeout(this.playbackTimeout);
}
this.audioDeltaList.length = 0;
yield this.pcmStreamPlayer.interrupt();
this.closeWs();
});
}
/**
* Append text to the speech buffer
* @param {string} message - Text message to convert to speech
*/
append(message) {
var _a;
(_a = this.ws) === null || _a === void 0 ? void 0 : _a.send({
id: `event-${Date.now()}`,
event_type: api_1.WebsocketsEventType.INPUT_TEXT_BUFFER_APPEND,
data: {
delta: message,
},
});
}
/**
* Complete the speech buffer and start processing
*/
complete() {
var _a;
(_a = this.ws) === null || _a === void 0 ? void 0 : _a.send({
id: `event-${Date.now()}`,
event_type: api_1.WebsocketsEventType.INPUT_TEXT_BUFFER_COMPLETE,
});
}
/**
* Append text and complete in a single call
* @param {string} message - Text message to convert to speech
*/
appendAndComplete(message) {
this.append(message);
this.complete();
}
/**
* Interrupt playback and disconnect
* @returns {Promise<void>}
*/
interrupt() {
return __awaiter(this, void 0, void 0, function* () {
yield this.disconnect();
this.emit('completed', undefined);
console.debug('[speech] playback completed', this.totalDuration);
});
}
/**
* Pause audio playback
* @returns {Promise<void>}
*/
pause() {
return __awaiter(this, void 0, void 0, function* () {
if (this.playbackTimeout) {
clearTimeout(this.playbackTimeout);
this.playbackTimeout = null;
}
if (this.playbackStartTime && !this.playbackPauseTime) {
this.playbackPauseTime = Date.now();
this.elapsedBeforePause +=
(this.playbackPauseTime - this.playbackStartTime) / 1000;
}
yield this.pcmStreamPlayer.pause();
});
}
/**
* Resume audio playback
* @returns {Promise<void>}
*/
resume() {
return __awaiter(this, void 0, void 0, function* () {
if (this.playbackPauseTime) {
this.playbackStartTime = Date.now();
this.playbackPauseTime = null;
// Update the timeout with remaining duration
if (this.playbackTimeout) {
clearTimeout(this.playbackTimeout);
}
const remaining = this.totalDuration - this.elapsedBeforePause;
this.playbackTimeout = setTimeout(() => {
this.emit('completed', undefined);
console.debug('[speech] playback completed', this.totalDuration);
this.playbackStartTime = null;
this.elapsedBeforePause = 0;
}, remaining * 1000);
}
yield this.pcmStreamPlayer.resume();
});
}
/**
* Toggle between play and pause states
* @returns {Promise<void>}
*/
togglePlay() {
return __awaiter(this, void 0, void 0, function* () {
if (this.isPlaying()) {
yield this.pause();
}
else {
yield this.resume();
}
});
}
/**
* Check if audio is currently playing
* @returns {boolean}
*/
isPlaying() {
return this.pcmStreamPlayer.isPlaying();
}
/**
* Register an event listener
* @param {string} event - Event name to listen for
* @param {Function} callback - Callback function
*/
on(event, callback) {
var _a;
if (!this.listeners.has(event)) {
this.listeners.set(event, new Set());
}
(_a = this.listeners.get(event)) === null || _a === void 0 ? void 0 : _a.add(callback);
}
/**
* Remove an event listener
* @param {string} event - Event name
* @param {Function} callback - Callback function to remove
*/
off(event, callback) {
var _a;
(_a = this.listeners.get(event)) === null || _a === void 0 ? void 0 : _a.delete(callback);
}
/**
* Close the WebSocket connection
* @private
*/
closeWs() {
var _a, _b;
if (((_a = this.ws) === null || _a === void 0 ? void 0 : _a.readyState) === 1) {
(_b = this.ws) === null || _b === void 0 ? void 0 : _b.close();
}
this.ws = null;
}
/**
* Emit an event to all registered listeners
* @param {string} event - Event name
* @param {CreateSpeechWsRes|undefined} data - Event data
* @private
*/
emit(event, data) {
var _a;
(_a = this.listeners.get(event)) === null || _a === void 0 ? void 0 : _a.forEach(callback => callback(data));
}
}
exports.WsSpeechClient = WsSpeechClient;
exports.default = WsSpeechClient;