UNPKG

@mymj/midjourney

Version:

Node.js client for the unofficial MidJourney API.

1,003 lines (942 loc) 28.3 kB
import { MJConfig, WaitMjEvent, MJMessage, LoadingHandler, MJEmit, OnModal, MJShorten, MJDescribe, } from "./interfaces"; import { MidjourneyApi } from "./midjourney.api"; import { content2progress, content2prompt, formatOptions, formatPrompts, nextNonce, uriToHash, componentsToHash, filenameToHash, } from "./utils"; import { VerifyHuman } from "./verify.human"; import WebSocket from "isomorphic-ws"; import type { Inflate } from 'zlib-sync'; import { TextDecoder } from 'node:util'; import * as zlib from 'zlib-sync'; export class WsMessage { ws: WebSocket; private closed = false; private event: Array<{ event: string; callback: (message: any) => void }> = []; private waitMjEvents: Map<string, WaitMjEvent> = new Map(); private skipMessageId: string[] = []; private reconnectTime: boolean[] = []; // private heartbeatTask: NodeJS.Timer | null = null private lastSequence = 0; private inflate: Inflate | null = null; private readonly textDecoder = new TextDecoder(); public UserId = ""; public connecting = false; constructor(public config: MJConfig, public MJApi: MidjourneyApi) { this.ws = new this.config.WebSocket(this.config.WsBaseUrl); this.inflate = new zlib.Inflate({ chunkSize: 65_535, to: 'string', }); this.connecting = true; this.ws.addEventListener("open", this.open.bind(this)); this.onSystem("messageCreate", this.onMessageCreate.bind(this)); this.onSystem("messageUpdate", this.onMessageUpdate.bind(this)); this.onSystem("messageDelete", this.onMessageDelete.bind(this)); this.onSystem("ready", this.onReady.bind(this)); this.onSystem("interactionSuccess", this.onInteractionSuccess.bind(this)); this.onSystem("modalCreate", this.onModalCreate.bind(this)); } private async heartbeat(num: number) { if (this.reconnectTime[num]) return; //check if ws is closed if (this.closed) return; if (this.ws.readyState !== this.ws.OPEN) { this.reconnect(); return; } this.log("heartbeat", this.lastSequence); this.ws.send( JSON.stringify({ op: 1, d: this.lastSequence, }) ); await this.timeout(1000 * 40); this.heartbeat(num); } // private heartbeat(interval: number) { // const nextInterval = interval * Math.random(); // this.log(`heartbeat`, `send discord heartbeat after ${Math.round(nextInterval / 1000)}s`); // if (this.closed) return; // this.heartbeatTask = setTimeout(() => { // if (this.ws.readyState === WebSocket.OPEN) { // this.ws.send( // JSON.stringify({ // op: 1, // d: this.lastSequence, // }), // ); // this.heartbeat(interval); // } else { // this.reconnect(); // } // }, nextInterval); // } close() { this.closed = true; this.ws.close(); } async checkWs() { if (this.closed) return; if (this.ws.readyState !== this.ws.OPEN) { this.reconnect(); await this.onceReady(); } } async isReady() { if (this.closed) return false; if (this.connecting) { await this.onceReady(); } else { await this.checkWs(); } return true; } async onceReady() { return new Promise((resolve) => { this.once("ready", (user) => { //print user nickname console.log(`🎊 ws ready!!! Hi: ${user.global_name}`); resolve(this); }); }); } //try reconnect reconnect() { if (this.closed) return; this.inflate = new zlib.Inflate({ chunkSize: 65_535, to: 'string', }); this.ws = new this.config.WebSocket(this.config.WsBaseUrl); this.connecting = true; this.lastSequence = 0; // clear previours heartbeat // if (this.heartbeatTask && typeof this.heartbeatTask === 'number') { // clearInterval(this.heartbeatTask) // this.heartbeatTask = null // } this.ws.addEventListener("open", this.open.bind(this)); } private decodeMessage(data: WebSocket.Data) { if (this.inflate) { const decompressable = new Uint8Array(data as ArrayBuffer); const l = decompressable.length; const flush = l >= 4 && decompressable[l - 4] === 0x00 && decompressable[l - 3] === 0x00 && decompressable[l - 2] === 0xff && decompressable[l - 1] === 0xff; this.inflate.push(Buffer.from(decompressable), flush ? zlib.Z_SYNC_FLUSH : zlib.Z_NO_FLUSH); if (this.inflate.err) { this.log(`${this.inflate.err}${this.inflate.msg ? `: ${this.inflate.msg}` : ''}`); return null; } if (!flush) { return null; } const { result } = this.inflate; if (!result) { return null; } try { return JSON.parse(typeof result === 'string' ? result : this.textDecoder.decode(result)); } catch (error) { return null; } } else { return data; } } // After opening ws private async open() { const num = this.reconnectTime.length; this.log("open.time", num); this.reconnectTime.push(false); this.ws.addEventListener("message", (event) => { const res = this.decodeMessage(event.data); this.parseMessage(res); }); this.ws.addEventListener("error", (event) => { this.reconnectTime[num] = true; this.reconnect(); }); this.ws.addEventListener("close", (event) => { this.reconnectTime[num] = true; this.reconnect(); }); this.connecting = false; this.auth(); setTimeout(() => { this.heartbeat(num); }, 1000 * 10); } // auth private auth() { this.ws.send( JSON.stringify({ op: 2, d: { token: this.config.SalaiToken, capabilities: 8189, properties: { os: "Mac OS X", browser: "Chrome", device: "", }, compress: false, }, }) ); } async timeout(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } private async messageCreate(message: any) { const { embeds, id, nonce, components, attachments, content } = message; const hash = componentsToHash(components); const contentNonce = this.getNonceFromContent(message.content); let isJobQueued = false; if (nonce) { // this.log("waiting start image or info or error"); // there could be components with 'cancel job' but sometime not this.updateMjEventIdByNonce(nonce, id); if (hash) { this.updateHashByNonce(nonce, hash); } if (embeds?.[0]) { const { color, description, title } = embeds[0]; switch (color) { case 16711680: //error if (title === "Action needed to continue") { if (!description.includes('Our AI moderators feel your prompt might be against our community standards.')) { return this.continue(message); } } else if (title === "Pending mod message") { return this.continue(message); } else if (title === "Tos not accepted") { return this.continue(message); } const error = new Error(description); this.EventError(id, error); return; case 16776960: //warning console.warn(description); break; case 16239475: // rich if (title === "Job queued") { isJobQueued = true; } break; default: if ( title?.includes("continue") && description?.includes("verify you're human") ) { //verify human await this.verifyHuman(message); return; } if (title?.includes("Invalid")) { //error const error = new Error(description); this.EventError(id, error); return; } } } if (content === 'Failed to process your command :c') { this.EventError(id, new Error(content)); return } if (content.includes('Are you sure you want to imagine') && components?.length) { this.EventError(id, new Error('Permutation is not supported yet, please try again one by one.')); return } } else { // some case error message create new message which do not have nonce // use message_reference to match previours message if (embeds?.[0]) { const { description, title } = embeds[0]; const { message_reference } = message; if (title?.includes("Invalid")) { const refId = message_reference?.message_id; if (refId) { //error const error = new Error(description); this.EventError(refId, error); return; } } } // vary region create is special without nonce if (contentNonce) { const event = this.getEventByNonce(contentNonce); if (event) { this.updateMjEventIdByNonce(contentNonce, id); if (hash) { this.updateHashByNonce(contentNonce, hash); } } // } else { // const event = this.getEventById(id); // if (!event && hash) { // event = this.getEventByHash(hash); // } // // somecase MJ throw finished image directly. // if (!event) { // event = this.getEventByContent(message.content); // } // if (event && !event.hash) { // this.updateMjEventIdByNonce(contentNonce, id); // if (hash) { // this.updateHashByNonce(contentNonce, hash); // } // } } } if (hash && attachments?.length > 0 && components?.length > 0) { this.done(message); return; } this.log(`isJobQueued: ${isJobQueued}`); if (isJobQueued) { this.processingImage(message, isJobQueued); } else { this.messageUpdate(message); } } private messageUpdate(message: any) { // this.log("messageUpdate", message); const { content, embeds, interaction = {}, nonce, id, components, attachments, } = message; if (!nonce) { const { name } = interaction; switch (name) { case "settings": this.emit("settings", message); return; case "describe": let uri = embeds?.[0]?.image?.url; if (this.config.ImageProxy !== "") { uri = uri.replace( "https://cdn.discordapp.com/", this.config.ImageProxy ); } const describe: MJDescribe = { id: id, flags: message.flags, descriptions: embeds?.[0]?.description.split("\n\n"), uri: uri, proxy_url: embeds?.[0]?.image?.proxy_url, options: formatOptions(components), }; this.emitMJ(id, describe); break; case "prefer remix": if (content != "") { this.emit("prefer-remix", content); } break; case "shorten": const shorten: MJShorten = { description: embeds?.[0]?.description, prompts: formatPrompts(embeds?.[0]?.description as string), options: formatOptions(components), id, flags: message.flags, }; this.emitMJ(id, shorten); break; case "info": this.emit("info", embeds?.[0]?.description); case "subscribe": this.emit("subscribe", components?.[0]?.components?.[0]); return; } } if (embeds?.[0]) { // message is not able to continue if (!attachments || !attachments.length || !components || !components.length) { const { description, title, color } = embeds[0]; switch (color) { case 16711680: //error const error = new Error(description); this.EventError(id, error); return; default: break; } if (title === "Duplicate images detected") { const error = new Error(description); this.EventError(id, error); return; } } } if (content) { this.processingImage(message); } } //interaction success private async onInteractionSuccess({ nonce, id, }: { nonce: string; id: string; }) { // this.log("interactionSuccess", nonce, id); const event = this.getEventByNonce(nonce); if (!event) { return; } // event.onmodal && event.onmodal(nonce, id); } //modal create private async onModalCreate({ nonce, id, custom_id, components, }: { nonce: string; id: string; custom_id: string; components: any; }) { this.log("modalCreate", nonce, id); const event = this.getEventByNonce(nonce); if (!event) { return; } const prompt_custom_id = components[0]?.components[0]?.custom_id; event.onmodal && event.onmodal(nonce, id, custom_id, prompt_custom_id); } private async onReady(user: any) { this.UserId = user.id; } private async onMessageCreate(message: any) { const { channel_id, author, interaction } = message; if (channel_id !== this.config.ChannelId) return; if (author?.id !== this.config.BotId) return; if (interaction && interaction.user.id !== this.UserId) return; // this.log("[messageCreate]", JSON.stringify(message)); this.messageCreate(message); } private async onMessageUpdate(message: any) { const { channel_id, author, interaction } = message; if (channel_id !== this.config.ChannelId) return; if (author?.id !== this.config.BotId) return; if (interaction && interaction.user.id !== this.UserId) return; // this.log("[messageUpdate]", JSON.stringify(message)); this.messageUpdate(message); } private async onMessageDelete(message: any) { const { channel_id, id } = message; if (channel_id !== this.config.ChannelId) return; // @ts-ignore for (const [key, value] of this.waitMjEvents.entries()) { if (value.id === id) { this.waitMjEvents.set(key, { ...value, del: true }); } } } // parse message from ws private parseMessage(msg: any) { const operate = msg.op as number; const type = msg.t as string if (!type) { this.log("event", JSON.stringify(msg)); return; } else { this.log("event", type); } if (!isNaN(msg.s)) { this.lastSequence = msg.s; } // if (operate === 10 && msg?.d?.heartbeat_interval) { // this.heartbeat(msg.d.heartbeat_interval!); // } const message = msg.d; if (message.channel_id === this.config.ChannelId) { this.log("message", JSON.stringify(msg)); } else if (!type.includes("MESSAGE_") && type !== 'READY') { this.log("operations", msg.t); } switch (type) { case "READY": if (message.session_id) { this.config.ActiveSessionId = message.session_id; } this.emitSystem("ready", message.user); break; case "INTERACTION_IFRAME_MODAL_CREATE": if (message.nonce) { this.emit(message.nonce, message); } break; case "INTERACTION_MODAL_CREATE": if (message.nonce) { this.emitSystem("modalCreate", message); } break; case "MESSAGE_CREATE": this.emitSystem("messageCreate", message); break; case "MESSAGE_UPDATE": this.emitSystem("messageUpdate", message); break; case "MESSAGE_DELETE": this.emitSystem("messageDelete", message); break; case "INTERACTION_SUCCESS": this.emitSystem("interactionSuccess", message); break; case "INTERACTION_CREATE": if (message.nonce) { this.emitSystem("interactionCreate", message); } break; default: break; } } //continue click appeal or Acknowledged private async continue(message: any) { const { components, id, flags, nonce } = message; const appeal = components[0]?.components[0]; this.log("appeal", appeal); if (appeal) { var newnonce = nextNonce(); const httpStatus = await this.MJApi.CustomApi({ msgId: id, customId: appeal.custom_id, flags, nonce: newnonce, }); this.log("appeal.httpStatus", httpStatus); if (httpStatus == 204) { //todo this.log("new nonce", newnonce); this.on(newnonce, (data) => { this.log("nonce data", data); this.emit(nonce, data); }); } } } private async verifyHuman(message: any) { const { HuggingFaceToken } = this.config; if (HuggingFaceToken === "" || !HuggingFaceToken) { this.log("HuggingFaceToken is empty"); return; } const { embeds, components, id, flags, nonce } = message; const uri = embeds[0].image.url; const categories = components[0].components; const classify = categories.map((c: any) => c.label); const verifyClient = new VerifyHuman(this.config); const category = await verifyClient.verify(uri, classify); if (category) { const custom_id = categories.find( (c: any) => c.label === category ).custom_id; var newnonce = nextNonce(); const httpStatus = await this.MJApi.CustomApi({ msgId: id, customId: custom_id, flags, nonce: newnonce, }); if (httpStatus == 204) { this.on(newnonce, (data) => { this.emit(nonce, data); }); } this.log("verifyHumanApi", httpStatus, custom_id, message.id); } } private EventError(id: string, error: Error) { const event = this.getEventById(id); if (!event) { return; } const eventMsg: MJEmit = { error, }; this.log("MJ ERROR:", error.message); this.emit(event.nonce, eventMsg); } private done(message: any) { const { content, id, attachments, components, flags } = message; const { url, proxy_url, width, height } = attachments[0]; let uri = url; if (this.config.ImageProxy !== "") { uri = uri.replace("https://cdn.discordapp.com/", this.config.ImageProxy); } let hash: string | undefined; if (components && components.length) { hash = componentsToHash(components); } if (!hash) { hash = uriToHash(url); } const MJmsg: MJMessage = { id, flags, content, hash: uriToHash(url), progress: "done", uri, proxy_url, options: formatOptions(components), width, height, }; this.filterMessages(MJmsg); return; } private processingImage(message: any, isJobQueued?: boolean) { const { content, id, attachments, flags, components } = message; let event: WaitMjEvent | undefined; let hash: string | undefined; let uri, proxy_url, width, height; if (components && components.length) { hash = componentsToHash(components); } if (attachments && attachments.length) { const { url, filename } = attachments[0]; uri = url; proxy_url = attachments[0].proxy_url; width = attachments[0].width; height = attachments[0].height; if (this.config.ImageProxy !== "") { uri = uri.replace("https://cdn.discordapp.com/", this.config.ImageProxy); } if (!hash) { hash = filenameToHash(filename); } } if (hash) { // found same hash event event = this.getEventByHash(hash); this.log('find event by hash', event); } // track event by msg id for imagine at the begining. if (!event) { event = this.getEventById(id); this.log('find old event by id', event); } if (event && content) { event.prompt = content; // update hash for the first time, in imagine case if (hash && !event.hash) { this.log('update event by hash', event.id, hash); this.updateHashByid(id, hash); event.hash = hash; } } // while MJ queued, this will tell the status const progress = isJobQueued ? 'Job queued' : content2progress(content); const MJmsg: MJMessage = { uri: uri, proxy_url: proxy_url, content: content, flags: flags, hash, options: components && components.length && formatOptions(components), progress, width, height, }; const eventMsg: MJEmit = { message: MJmsg, }; if (event) { this.emitImage(event.nonce, eventMsg); } else { this.emitImage('OTHER_MSG', eventMsg); } } private async filterMessages(MJmsg: MJMessage) { // delay 300ms for discord message delete await this.timeout(300); let event: WaitMjEvent | undefined; if (MJmsg.hash) { event = this.getEventByHash(MJmsg.hash); } // somecase MJ throw finished image directly. if (!event) { event = this.getEventByContent(MJmsg.content); } const eventMsg: MJEmit = { message: MJmsg, }; if (!event) { this.log("FilterMessages not found", MJmsg.hash, this.waitMjEvents); this.emitImage('OTHER_MSG', eventMsg); return; } this.emitImage(event.nonce, eventMsg); } private getEventByContent(content: string) { const prompt = content2prompt(content); //fist del message // @ts-ignore for (const [key, value] of this.waitMjEvents.entries()) { const prompCache = content2prompt(value?.prompt as string); if ( value.del === true && prompCache.includes(prompt) ) { return value as WaitMjEvent; } } // @ts-ignore for (const [key, value] of this.waitMjEvents.entries()) { const prompCache = content2prompt(value?.prompt as string); if (prompCache.includes(prompt)) { return value as WaitMjEvent; } } } private getEventByHash(hash: string) { // @ts-ignore for (const [key, value] of this.waitMjEvents.entries()) { if (value.hash === hash) { return value as WaitMjEvent; } } } private getEventById(id: string) { // @ts-ignore for (const [key, value] of this.waitMjEvents.entries()) { if (value.id === id) { return value as WaitMjEvent; } } } private getEventByNonce(nonce: string) { // @ts-ignore for (const [key, value] of this.waitMjEvents.entries()) { if (value.nonce === nonce) { return value as WaitMjEvent; } } } private getNonceFromContent(content: string) { return content.match(/\[(.*?)\]/)?.[1]; } private updateMjEventIdByNonce(nonce: string, id: string) { if (nonce === "" || id === "") return; let event = this.waitMjEvents.get(nonce); if (!event) return; event.id = id; this.log("updateMjEventIdByNonce success", event); return event; } private updateHashByNonce(nonce: string, hash: string) { if (nonce === "" || hash === "") return; let event = this.waitMjEvents.get(nonce); if (!event) return; event.hash = hash; this.log("updateHashByNonce success", event); return event; } private updateHashByid(id: string, hash: string) { if (id === "" || hash === "") return; let event = this.getEventById(id); if (!event) return; event.hash = hash; this.log("updateHashByMjEventId success", event); return event; } protected async log(...args: any[]) { this.config.Debug && console.info(...args, new Date().toISOString()); } emit(event: string, message: any) { this.event .filter((e) => e.event === event) .forEach((e) => e.callback(message)); } private emitImage(type: string, message: MJEmit) { this.emit(type, message); } //FIXME: emitMJ rename private emitMJ(id: string, data: any) { const event = this.getEventById(id); if (!event) return; this.emit(event.nonce, data); } on(event: string, callback: (message: any) => void) { this.event.push({ event, callback }); } onSystem( event: | "ready" | "messageCreate" | "messageUpdate" | "messageDelete" | "modalCreate" | "interactionCreate" | "interactionSuccess", callback: (message: any) => void ) { this.on(event, callback); } private emitSystem( type: | "ready" | "messageCreate" | "messageUpdate" | "messageDelete" | "modalCreate" | "interactionSuccess" | "interactionCreate", message: MJEmit ) { this.emit(type, message); } once(event: string, callback: (message: any) => void) { const once = (message: any) => { this.remove(event, once); callback(message); }; this.event.push({ event, callback: once }); } remove(event: string, callback: (message: any) => void) { this.event = this.event.filter( (e) => e.event !== event && e.callback !== callback ); } removeEvent(event: string) { this.event = this.event.filter((e) => e.event !== event); } onceMJ(nonce: string, callback: (data: any) => void) { const once = (message: any) => { this.remove(nonce, once); //FIXME: removeWaitMjEvent this.removeWaitMjEvent(nonce); callback(message); }; //FIXME: addWaitMjEvent this.waitMjEvents.set(nonce, { nonce }); this.event.push({ event: nonce, callback: once }); } private removeSkipMessageId(messageId: string) { const index = this.skipMessageId.findIndex((id) => id !== messageId); if (index !== -1) { this.skipMessageId.splice(index, 1); } } private removeWaitMjEvent(nonce: string) { this.waitMjEvents.delete(nonce); } onceImage(nonce: string, callback: (data: MJEmit) => void) { const once = (data: MJEmit) => { const { message, error } = data; if (error || (message && message.progress === "done")) { this.remove(nonce, once); } callback(data); }; this.event.push({ event: nonce, callback: once }); } async waitImageMessage({ nonce, prompt, onmodal, messageId, loading, }: { nonce: string; prompt?: string; messageId?: string; onmodal?: OnModal; loading?: LoadingHandler; }) { if (messageId) this.skipMessageId.push(messageId); return new Promise<MJMessage | null>((resolve, reject) => { const handleImageMessage = ({ message, error }: MJEmit) => { if (error) { this.removeWaitMjEvent(nonce); reject(error); return; } if (message && message.progress === "done") { this.removeWaitMjEvent(nonce); messageId && this.removeSkipMessageId(messageId); resolve(message); return; } message && loading && loading({ uri: message.uri, progress: message.progress || "", options: message.options, hash: message.hash, }); }; this.waitMjEvents.set(nonce, { nonce, prompt, onmodal: async (oldnonce, id, custom_id, prompt_custom_id) => { if (onmodal === undefined) { // reject(new Error("onmodal is not defined")) return ""; } var nonce = await onmodal(oldnonce, id, custom_id, prompt_custom_id); if (nonce === "") { // reject(new Error("onmodal return empty nonce")) return ""; } this.removeWaitMjEvent(oldnonce); this.waitMjEvents.set(nonce, { nonce }); this.onceImage(nonce, handleImageMessage); return nonce; }, }); this.onceImage(nonce, handleImageMessage); }); } async waitOnceMJ(nonce: string) { return new Promise<any>((resolve) => { this.onceMJ(nonce, (message) => { resolve(message); }); }); } async waitOnce(nonce: string) { return new Promise<any>((resolve) => { this.once(nonce, (message) => { resolve(message); }); }); } }