@coze/uniapp-api
Version:
Official Coze UniApp SDK for seamless AI integration into your applications | 扣子官方 UniApp SDK，助您轻松集成 AI 能力到应用中
680 lines (679 loc) • 25.9 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.PcmStreamPlayer = void 0;
/* eslint-disable @typescript-eslint/no-explicit-any */
const resampler_1 = require("./resampler");
const g711_1 = require("./codecs/g711");
/**
* PcmStreamPlayer for WeChat Mini Program
* Plays audio streams received in raw PCM16, G.711a, or G.711u chunks
* @class
*/
class PcmStreamPlayer {
/**
* Creates a new PcmStreamPlayer instance
* @param {{sampleRate?: number, defaultFormat?: AudioFormat}} options
* @returns {PcmStreamPlayer}
*/
constructor({ sampleRate = 24000, defaultFormat = 'pcm', volume = 1.0, } = {}) {
Object.defineProperty(this, "audioContext", {
enumerable: true,
configurable: true,
writable: true,
value: null
}); // Using 'any' for WebAudioContext due to type limitations
Object.defineProperty(this, "inputSampleRate", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "outputSampleRate", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "audioQueue", {
enumerable: true,
configurable: true,
writable: true,
value: []
});
Object.defineProperty(this, "volume", {
enumerable: true,
configurable: true,
writable: true,
value: 1.0
}); // Volume level from 0.0 (muted) to 1.0 (full volume)
Object.defineProperty(this, "trackSampleOffsets", {
enumerable: true,
configurable: true,
writable: true,
value: {}
});
Object.defineProperty(this, "interruptedTrackIds", {
enumerable: true,
configurable: true,
writable: true,
value: {}
});
Object.defineProperty(this, "isInitialized", {
enumerable: true,
configurable: true,
writable: true,
value: false
});
Object.defineProperty(this, "isProcessing", {
enumerable: true,
configurable: true,
writable: true,
value: false
});
Object.defineProperty(this, "scriptNode", {
enumerable: true,
configurable: true,
writable: true,
value: null
});
Object.defineProperty(this, "bufferSize", {
enumerable: true,
configurable: true,
writable: true,
value: 1024
});
Object.defineProperty(this, "base64Queue", {
enumerable: true,
configurable: true,
writable: true,
value: []
});
Object.defineProperty(this, "isProcessingQueue", {
enumerable: true,
configurable: true,
writable: true,
value: false
});
Object.defineProperty(this, "lastAudioProcessTime", {
enumerable: true,
configurable: true,
writable: true,
value: Infinity
});
Object.defineProperty(this, "processingTimeThreshold", {
enumerable: true,
configurable: true,
writable: true,
value: 0
}); // 1ms threshold
Object.defineProperty(this, "isPaused", {
enumerable: true,
configurable: true,
writable: true,
value: false
}); // Track pause state
// Current buffer and position for continuous playback
Object.defineProperty(this, "currentBuffer", {
enumerable: true,
configurable: true,
writable: true,
value: null
});
Object.defineProperty(this, "playbackPosition", {
enumerable: true,
configurable: true,
writable: true,
value: 0
});
/**
* Default audio format
*/
Object.defineProperty(this, "defaultFormat", {
enumerable: true,
configurable: true,
writable: true,
value: 'pcm'
});
this.inputSampleRate = sampleRate;
// 微信小程序，输出的采样率是固定的，所有需要重采样
this.outputSampleRate = PcmStreamPlayer.getSampleRate();
this.defaultFormat = defaultFormat;
this.setVolume(volume);
}
/**
* Initialize the audio context
* @private
* @returns {boolean}
*/
initialize() {
if (this.isInitialized) {
return true;
}
try {
// Create WebAudioContext for UniApp environment
this.audioContext = uni.createWebAudioContext();
if (!this.audioContext) {
console.error('Failed to create WebAudioContext');
return false;
}
// 在下一帧音频处理前5ms，确保主线程是空闲的
this.processingTimeThreshold =
Math.floor((this.bufferSize / this.audioContext.sampleRate) * 1000) - 5;
// Initialize silent audio trigger for iPhone
this.initSilentModeTrigger();
this.isInitialized = true;
return true;
}
catch (error) {
console.error('Error initializing audio context:', error);
return false;
}
}
/**
* Initialize a silent audio player to bypass iPhone silent mode
* @private
*/
initSilentModeTrigger() {
try {
// Only initialize once
if (!PcmStreamPlayer.triggerAudio) {
uni.setInnerAudioOption({
obeyMuteSwitch: false, // 设置为false以忽略静音开关
success: () => {
console.log('Inner audio option set obeyMuteSwitch=false successfully');
},
fail: err => {
console.error('Failed to set inner audio option:', err);
},
});
// Create and configure the trigger audio
const triggerAudio = uni.createInnerAudioContext();
// Use a base64 silent audio as a temporary placeholder until loaded
// In real implementation, should use a file in the app package
triggerAudio.src =
'data:audio/mp3;base64,SUQzBAAAAAAAI1RTU0UAAAAPAAADTGF2ZjU4Ljc2LjEwMAAAAAAAAAAAAAAA//tQwAAAAAAAAAAAAAAAAAAAAAAASW5mbwAAAA8AAAACAAABIADAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDV1dXV1dXV1dXV1dXV1dXV1dXV1dXV1dXV6urq6urq6urq6urq6urq6urq6urq6urq6v////////////////////////////////8AAAAATGF2YzU4LjEzAAAAAAAAAAAAAAAAJAAAAAAAAAAAASAthz7PAAAAAAAAAAAAAAAA//tAwAAABpADjMQAACK2IHbYwggI0JMZ4M8y5wPEI7iSHf5DMjMH5QdHI25QZIguRmDIJnoZgyDGfCUGQdBjGDa+jm7aGaABBAEAghzNIJhJRmCEYbJkHmUCuMY1/AAIAAQACAQ8QDSSQTJ7gICAwQkDSiYVBpgoBnJDQA==';
triggerAudio.loop = true;
triggerAudio.volume = 0.01; // Set to very low volume
triggerAudio.obeyMuteSwitch = false;
PcmStreamPlayer.triggerAudio = triggerAudio;
// Play immediately to bypass iPhone silent mode
triggerAudio.onPlay = () => {
console.log('Trigger audio played');
};
triggerAudio.play();
}
}
catch (error) {
console.error('Error initializing silent mode trigger:', error);
}
}
/**
* Start audio playback system
* @private
*/
startPlayback() {
var _a, _b;
try {
// Initialize audio if needed
if (!this.isInitialized) {
const initialized = this.initialize();
if (!initialized) {
return false;
}
}
// If there is no data in the queue, exit
if (this.audioQueue.length === 0) {
return false;
}
// Clean up any existing scriptNode to prevent duplicate audio playback
if (this.scriptNode) {
try {
this.scriptNode.disconnect();
this.scriptNode = null;
}
catch (error) {
console.warn('Error disconnecting previous script node:', error);
}
}
// Using optional chaining to handle potential null and checking if method exists
const scriptNode = ((_a = this.audioContext) === null || _a === void 0 ? void 0 : _a.createScriptProcessor)
? this.audioContext.createScriptProcessor(this.bufferSize, 0, 1)
: null;
if (!scriptNode) {
console.error('Failed to create script processor node');
return false;
}
this.scriptNode = scriptNode;
this.isProcessing = true;
// Process audio data
scriptNode.onaudioprocess = (e) => {
const outputBuffer = e.outputBuffer.getChannelData(0);
// Fill the output buffer
this.fillOutputBuffer(outputBuffer);
// Record the last audio process time
this.lastAudioProcessTime = Date.now();
// Check if we have base64 data to process and not currently processing
this.processBase64Queue();
};
// Connect to destination (speakers)
scriptNode.connect((_b = this.audioContext) === null || _b === void 0 ? void 0 : _b.destination);
return true;
}
catch (error) {
console.error('Error starting audio playback:', error);
return false;
}
}
/**
* Fill the output buffer with audio data from the current buffer or queue
* @private
*/
fillOutputBuffer(outputBuffer) {
if (!this.currentBuffer) {
if (!this.getNextBuffer()) {
// No data available, fill with silence
for (let i = 0; i < outputBuffer.length; i++) {
outputBuffer[i] = 0;
}
this.isProcessing = false;
return;
}
}
// We have a buffer, process it with volume control
for (let i = 0; i < outputBuffer.length; i++) {
if (this.currentBuffer &&
this.playbackPosition < this.currentBuffer.length) {
// Apply volume scaling to the PCM samples (convert Int16 to Float32 and scale by volume)
outputBuffer[i] =
this.volume <= 0
? 0
: (this.currentBuffer[this.playbackPosition] / 0x8000) *
this.volume;
this.playbackPosition++;
}
else {
// Current buffer is exhausted, try to get next buffer
if (!this.getNextBuffer()) {
// No more buffers available
outputBuffer[i] = 0;
this.isProcessing = i !== outputBuffer.length - 1;
}
else {
// Got a new buffer, process it
if (this.currentBuffer) {
// Apply volume scaling
outputBuffer[i] =
this.volume <= 0
? 0
: (this.currentBuffer[this.playbackPosition] / 0x8000) *
this.volume;
this.playbackPosition++;
}
else {
// Should not happen but still handle it
outputBuffer[i] = 0;
}
this.isProcessing = true;
}
}
}
}
/**
* Get the next buffer from the queue and prepare it for playback
* @private
* @returns {boolean} True if a new buffer was prepared, false if queue is empty
*/
getNextBuffer() {
// If queue is empty, return false
if (this.audioQueue.length === 0) {
return false;
}
// Get the next PCM data from queue
const pcmData = this.audioQueue.shift();
if (!pcmData || pcmData.length === 0) {
this.currentBuffer = null;
return false;
}
// Keep the data in Int16Array format and only convert when needed during playback
this.currentBuffer = pcmData;
this.playbackPosition = 0;
return true;
}
/**
* Pauses audio playback
*/
pause() {
return __awaiter(this, void 0, void 0, function* () {
var _a, _b;
if (this.audioContext && !this.isPaused) {
try {
// In UniApp environment, suspend the audio context
// Check if state and suspend methods exist
if (((_a = this.audioContext) === null || _a === void 0 ? void 0 : _a.state) === 'running' &&
typeof ((_b = this.audioContext) === null || _b === void 0 ? void 0 : _b.suspend) === 'function') {
yield this.audioContext.suspend();
}
this.isPaused = true;
}
catch (error) {
console.error('Error pausing audio playback:', error);
}
}
});
}
/**
* Resumes audio playback
*/
resume() {
return __awaiter(this, void 0, void 0, function* () {
var _a, _b;
if (this.audioContext && this.isPaused) {
try {
// In UniApp environment, resume the audio context
// Check if state and resume methods exist
if (((_a = this.audioContext) === null || _a === void 0 ? void 0 : _a.state) === 'suspended' &&
typeof ((_b = this.audioContext) === null || _b === void 0 ? void 0 : _b.resume) === 'function') {
yield this.audioContext.resume();
}
this.isPaused = false;
// If no scriptNode exists, start playback again
if (!this.scriptNode && this.audioQueue.length > 0) {
yield this.startPlayback();
}
}
catch (error) {
console.error('Error resuming audio playback:', error);
}
}
});
}
/**
* Toggles between play and pause states
*/
togglePlay() {
return __awaiter(this, void 0, void 0, function* () {
if (this.isPaused) {
yield this.resume();
}
else {
yield this.pause();
}
});
}
/**
* Checks if audio is currently playing
* @returns {boolean}
*/
isPlaying() {
var _a;
return Boolean(this.audioContext &&
!this.isPaused &&
(this.isProcessing || this.audioQueue.length > 0) &&
((_a = this.audioContext) === null || _a === void 0 ? void 0 : _a.state) === 'running');
}
isProcessingIdle() {
if (this.lastAudioProcessTime === 0) {
return true;
}
const now = Date.now();
const diff = now - this.lastAudioProcessTime;
if (diff > 100 || diff < this.processingTimeThreshold) {
return true;
}
return false;
}
/**
* Adds base64 encoded PCM data to a queue for processing
* This prevents blocking the main thread during audio processing
* @param {string} base64String - Base64 encoded PCM data
* @param {string} trackId - Track identifier
* @returns {boolean} - Success status
*/
addBase64PCM(base64String, trackId = 'default') {
// Add to processing queue
this.base64Queue.push({ base64String, trackId });
// If we're outside the processing window, try to process the queue
if (this.isProcessingIdle()) {
this.processBase64Queue();
}
return true;
}
/**
* Process the base64 queue when the main thread is idle
* @private
*/
processBase64Queue() {
// If already processing or queue is empty, do nothing
if (this.isProcessingQueue || this.base64Queue.length === 0) {
return;
}
this.isProcessingQueue = true;
try {
// Process one item from the queue
const item = this.base64Queue.shift();
if (item) {
const { base64String, trackId } = item;
// console.log(
// `Processing base64 data for track ${trackId}, queue size: ${this.base64Queue.length}`,
// );
const binaryString = uni.base64ToArrayBuffer(base64String);
this.add16BitPCM(binaryString, trackId);
}
}
catch (error) {
console.error('Error processing base64 queue:', error);
}
finally {
this.isProcessingQueue = false;
// If there are more items and we're still outside the processing window, process the next item
if (this.base64Queue.length > 0) {
if (this.isProcessingIdle()) {
// Use setTimeout to give the main thread a chance to breathe
setTimeout(() => this.processBase64Queue(), 0);
}
}
}
}
/**
* Adds audio data to the currently playing audio stream
* @param {ArrayBuffer|Int16Array|Uint8Array} arrayBuffer
* @param {string} [trackId]
* @param {AudioFormat} [format] - Audio format: 'pcm', 'g711a', or 'g711u'
* @returns {Int16Array}
*/
add16BitPCM(arrayBuffer, trackId = 'default', format) {
if (typeof trackId !== 'string') {
throw new Error('trackId must be a string');
}
else if (this.interruptedTrackIds[trackId]) {
return new Int16Array();
}
let buffer;
const audioFormat = format || this.defaultFormat;
if (arrayBuffer instanceof Int16Array) {
// Already in PCM format
buffer = arrayBuffer;
}
else if (arrayBuffer instanceof Uint8Array) {
// Handle different formats based on the specified format
if (audioFormat === 'g711a') {
buffer = (0, g711_1.decodeAlaw)(arrayBuffer);
}
else if (audioFormat === 'g711u') {
buffer = (0, g711_1.decodeUlaw)(arrayBuffer);
}
else {
// Treat as PCM data in Uint8Array
buffer = new Int16Array(arrayBuffer.buffer);
}
}
else if (arrayBuffer instanceof ArrayBuffer) {
// Handle different formats based on the specified format
if (audioFormat === 'g711a') {
buffer = (0, g711_1.decodeAlaw)(new Uint8Array(arrayBuffer));
}
else if (audioFormat === 'g711u') {
buffer = (0, g711_1.decodeUlaw)(new Uint8Array(arrayBuffer));
}
else {
// Default to PCM
buffer = new Int16Array(arrayBuffer);
}
}
else {
throw new Error('argument must be Int16Array, Uint8Array, or ArrayBuffer');
}
// Resample the buffer if input and output sample rates are different
if (this.inputSampleRate !== this.outputSampleRate) {
buffer = resampler_1.Resampler.resample(buffer, this.inputSampleRate, this.outputSampleRate);
}
// Add to the audio queue
this.audioQueue.push(buffer);
if (!this.isProcessing && !this.isPaused) {
this.startPlayback();
}
return buffer;
}
/**
* Gets the offset (sample count) of the currently playing stream
* @param {boolean} [interrupt]
* @returns {{trackId: string|null, offset: number, currentTime: number} | null}
*/
getTrackSampleOffset(interrupt = false) {
var _a;
if (!this.audioContext) {
return null;
}
// Calculate approximate offset based on audio context time
const currentTime = ((_a = this.audioContext) === null || _a === void 0 ? void 0 : _a.currentTime) || 0;
const offset = Math.floor(currentTime * this.inputSampleRate);
const requestId = Date.now().toString();
const trackId = 'default'; // We're using a default track for all audio
const result = {
trackId,
offset,
currentTime,
};
this.trackSampleOffsets[requestId] = result;
if (interrupt && trackId) {
this.interruptedTrackIds[trackId] = true;
// Clear the audio queue for interrupted track
this.audioQueue = [];
// Disconnect the current scriptNode if exists
if (this.scriptNode) {
try {
this.scriptNode.disconnect();
this.scriptNode = null;
this.currentBuffer = null;
this.playbackPosition = 0;
this.isPaused = false;
}
catch (error) {
console.warn('Error disconnecting script node:', error);
}
}
this.isProcessing = false;
}
return result;
}
/**
* Strips the current stream and returns the sample offset of the audio
* @returns {{trackId: string|null, offset: number, currentTime: number} | null}
*/
interrupt() {
// Clear current buffer and reset playback position
this.currentBuffer = null;
this.playbackPosition = 0;
return this.getTrackSampleOffset(true);
}
/**
* Set the input sample rate for audio playback
* @param {number} sampleRate - The sample rate of the incoming audio data
*/
setSampleRate(sampleRate) {
// We can change the input sample rate at any time
this.inputSampleRate = sampleRate;
console.log(`Input sample rate set to ${sampleRate}Hz, output sample rate is ${this.outputSampleRate}Hz`);
}
/**
* Set the default audio format
* @param {AudioFormat} format
*/
setDefaultFormat(format) {
this.defaultFormat = format;
}
/**
* Adds G.711 A-law encoded audio data to the currently playing audio stream
* @param {ArrayBuffer|Uint8Array} arrayBuffer - G.711 A-law encoded data
* @param {string} [trackId]
* @returns {Int16Array}
*/
addG711a(arrayBuffer, trackId = 'default') {
return this.add16BitPCM(arrayBuffer, trackId, 'g711a');
}
/**
* Adds G.711 μ-law encoded audio data to the currently playing audio stream
* @param {ArrayBuffer|Uint8Array} arrayBuffer - G.711 μ-law encoded data
* @param {string} [trackId]
* @returns {Int16Array}
*/
addG711u(arrayBuffer, trackId = 'default') {
return this.add16BitPCM(arrayBuffer, trackId, 'g711u');
}
/**
* Get the WebAudioContext's sample rate
* @returns {number} System sample rate
* @static
*/
static getSampleRate() {
const audioContext = uni.createWebAudioContext();
const { sampleRate } = audioContext;
audioContext.close();
return sampleRate;
}
/**
* Set volume level for audio playback
* @param {number} volume - Volume level from 0.0 (muted) to 1.0 (full volume)
*/
setVolume(volume) {
// Ensure volume is between 0 and 1
this.volume = Math.max(0, Math.min(1, volume));
console.log(`Audio volume set to ${this.volume}`);
}
/**
* Get current volume level
* @returns {number} Current volume level from 0.0 (muted) to 1.0 (full volume)
*/
getVolume() {
return this.volume;
}
/**
* Cleanup static resources when the app is closed or the page is unloaded
*/
static cleanup() {
if (PcmStreamPlayer.triggerAudio) {
PcmStreamPlayer.triggerAudio.stop();
PcmStreamPlayer.triggerAudio.destroy();
PcmStreamPlayer.triggerAudio = null;
}
}
}
exports.PcmStreamPlayer = PcmStreamPlayer;
// Trigger audio for iPhone silent mode
Object.defineProperty(PcmStreamPlayer, "triggerAudio", {
enumerable: true,
configurable: true,
writable: true,
value: null
});
// Export for use with import {PcmStreamPlayer} from '@coze/uniapp-api/ws-tools'
exports.default = PcmStreamPlayer;