@mymj/midjourney
Version:
Node.js client for the unofficial MidJourney API.
1,003 lines (942 loc) • 28.3 kB
text/typescript
import {
MJConfig,
WaitMjEvent,
MJMessage,
LoadingHandler,
MJEmit,
OnModal,
MJShorten,
MJDescribe,
} from "./interfaces";
import { MidjourneyApi } from "./midjourney.api";
import {
content2progress,
content2prompt,
formatOptions,
formatPrompts,
nextNonce,
uriToHash,
componentsToHash,
filenameToHash,
} from "./utils";
import { VerifyHuman } from "./verify.human";
import WebSocket from "isomorphic-ws";
import type { Inflate } from 'zlib-sync';
import { TextDecoder } from 'node:util';
import * as zlib from 'zlib-sync';
export class WsMessage {
ws: WebSocket;
private closed = false;
private event: Array<{ event: string; callback: (message: any) => void }> =
[];
private waitMjEvents: Map<string, WaitMjEvent> = new Map();
private skipMessageId: string[] = [];
private reconnectTime: boolean[] = [];
// private heartbeatTask: NodeJS.Timer | null = null
private lastSequence = 0;
private inflate: Inflate | null = null;
private readonly textDecoder = new TextDecoder();
public UserId = "";
public connecting = false;
constructor(public config: MJConfig, public MJApi: MidjourneyApi) {
this.ws = new this.config.WebSocket(this.config.WsBaseUrl);
this.inflate = new zlib.Inflate({
chunkSize: 65_535,
to: 'string',
});
this.connecting = true;
this.ws.addEventListener("open", this.open.bind(this));
this.onSystem("messageCreate", this.onMessageCreate.bind(this));
this.onSystem("messageUpdate", this.onMessageUpdate.bind(this));
this.onSystem("messageDelete", this.onMessageDelete.bind(this));
this.onSystem("ready", this.onReady.bind(this));
this.onSystem("interactionSuccess", this.onInteractionSuccess.bind(this));
this.onSystem("modalCreate", this.onModalCreate.bind(this));
}
private async heartbeat(num: number) {
if (this.reconnectTime[num]) return;
//check if ws is closed
if (this.closed) return;
if (this.ws.readyState !== this.ws.OPEN) {
this.reconnect();
return;
}
this.log("heartbeat", this.lastSequence);
this.ws.send(
JSON.stringify({
op: 1,
d: this.lastSequence,
})
);
await this.timeout(1000 * 40);
this.heartbeat(num);
}
// private heartbeat(interval: number) {
// const nextInterval = interval * Math.random();
// this.log(`heartbeat`, `send discord heartbeat after ${Math.round(nextInterval / 1000)}s`);
// if (this.closed) return;
// this.heartbeatTask = setTimeout(() => {
// if (this.ws.readyState === WebSocket.OPEN) {
// this.ws.send(
// JSON.stringify({
// op: 1,
// d: this.lastSequence,
// }),
// );
// this.heartbeat(interval);
// } else {
// this.reconnect();
// }
// }, nextInterval);
// }
close() {
this.closed = true;
this.ws.close();
}
async checkWs() {
if (this.closed) return;
if (this.ws.readyState !== this.ws.OPEN) {
this.reconnect();
await this.onceReady();
}
}
async isReady() {
if (this.closed) return false;
if (this.connecting) {
await this.onceReady();
} else {
await this.checkWs();
}
return true;
}
async onceReady() {
return new Promise((resolve) => {
this.once("ready", (user) => {
//print user nickname
console.log(`🎊 ws ready!!! Hi: ${user.global_name}`);
resolve(this);
});
});
}
//try reconnect
reconnect() {
if (this.closed) return;
this.inflate = new zlib.Inflate({
chunkSize: 65_535,
to: 'string',
});
this.ws = new this.config.WebSocket(this.config.WsBaseUrl);
this.connecting = true;
this.lastSequence = 0;
// clear previours heartbeat
// if (this.heartbeatTask && typeof this.heartbeatTask === 'number') {
// clearInterval(this.heartbeatTask)
// this.heartbeatTask = null
// }
this.ws.addEventListener("open", this.open.bind(this));
}
private decodeMessage(data: WebSocket.Data) {
if (this.inflate) {
const decompressable = new Uint8Array(data as ArrayBuffer);
const l = decompressable.length;
const flush =
l >= 4 &&
decompressable[l - 4] === 0x00 &&
decompressable[l - 3] === 0x00 &&
decompressable[l - 2] === 0xff &&
decompressable[l - 1] === 0xff;
this.inflate.push(Buffer.from(decompressable), flush ? zlib.Z_SYNC_FLUSH : zlib.Z_NO_FLUSH);
if (this.inflate.err) {
this.log(`${this.inflate.err}${this.inflate.msg ? `: ${this.inflate.msg}` : ''}`);
return null;
}
if (!flush) {
return null;
}
const { result } = this.inflate;
if (!result) {
return null;
}
try {
return JSON.parse(typeof result === 'string' ? result : this.textDecoder.decode(result));
} catch (error) {
return null;
}
} else {
return data;
}
}
// After opening ws
private async open() {
const num = this.reconnectTime.length;
this.log("open.time", num);
this.reconnectTime.push(false);
this.ws.addEventListener("message", (event) => {
const res = this.decodeMessage(event.data);
this.parseMessage(res);
});
this.ws.addEventListener("error", (event) => {
this.reconnectTime[num] = true;
this.reconnect();
});
this.ws.addEventListener("close", (event) => {
this.reconnectTime[num] = true;
this.reconnect();
});
this.connecting = false;
this.auth();
setTimeout(() => {
this.heartbeat(num);
}, 1000 * 10);
}
// auth
private auth() {
this.ws.send(
JSON.stringify({
op: 2,
d: {
token: this.config.SalaiToken,
capabilities: 8189,
properties: {
os: "Mac OS X",
browser: "Chrome",
device: "",
},
compress: false,
},
})
);
}
async timeout(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
private async messageCreate(message: any) {
const { embeds, id, nonce, components, attachments, content } = message;
const hash = componentsToHash(components);
const contentNonce = this.getNonceFromContent(message.content);
let isJobQueued = false;
if (nonce) {
// this.log("waiting start image or info or error");
// there could be components with 'cancel job' but sometime not
this.updateMjEventIdByNonce(nonce, id);
if (hash) {
this.updateHashByNonce(nonce, hash);
}
if (embeds?.[0]) {
const { color, description, title } = embeds[0];
switch (color) {
case 16711680: //error
if (title === "Action needed to continue") {
if (!description.includes('Our AI moderators feel your prompt might be against our community standards.')) {
return this.continue(message);
}
} else if (title === "Pending mod message") {
return this.continue(message);
} else if (title === "Tos not accepted") {
return this.continue(message);
}
const error = new Error(description);
this.EventError(id, error);
return;
case 16776960: //warning
console.warn(description);
break;
case 16239475: // rich
if (title === "Job queued") {
isJobQueued = true;
}
break;
default:
if (
title?.includes("continue") &&
description?.includes("verify you're human")
) {
//verify human
await this.verifyHuman(message);
return;
}
if (title?.includes("Invalid")) {
//error
const error = new Error(description);
this.EventError(id, error);
return;
}
}
}
if (content === 'Failed to process your command :c') {
this.EventError(id, new Error(content));
return
}
if (content.includes('Are you sure you want to imagine') && components?.length) {
this.EventError(id, new Error('Permutation is not supported yet, please try again one by one.'));
return
}
} else {
// some case error message create new message which do not have nonce
// use message_reference to match previours message
if (embeds?.[0]) {
const { description, title } = embeds[0];
const { message_reference } = message;
if (title?.includes("Invalid")) {
const refId = message_reference?.message_id;
if (refId) {
//error
const error = new Error(description);
this.EventError(refId, error);
return;
}
}
}
// vary region create is special without nonce
if (contentNonce) {
const event = this.getEventByNonce(contentNonce);
if (event) {
this.updateMjEventIdByNonce(contentNonce, id);
if (hash) {
this.updateHashByNonce(contentNonce, hash);
}
}
// } else {
// const event = this.getEventById(id);
// if (!event && hash) {
// event = this.getEventByHash(hash);
// }
// // somecase MJ throw finished image directly.
// if (!event) {
// event = this.getEventByContent(message.content);
// }
// if (event && !event.hash) {
// this.updateMjEventIdByNonce(contentNonce, id);
// if (hash) {
// this.updateHashByNonce(contentNonce, hash);
// }
// }
}
}
if (hash && attachments?.length > 0 && components?.length > 0) {
this.done(message);
return;
}
this.log(`isJobQueued: ${isJobQueued}`);
if (isJobQueued) {
this.processingImage(message, isJobQueued);
} else {
this.messageUpdate(message);
}
}
private messageUpdate(message: any) {
// this.log("messageUpdate", message);
const {
content,
embeds,
interaction = {},
nonce,
id,
components,
attachments,
} = message;
if (!nonce) {
const { name } = interaction;
switch (name) {
case "settings":
this.emit("settings", message);
return;
case "describe":
let uri = embeds?.[0]?.image?.url;
if (this.config.ImageProxy !== "") {
uri = uri.replace(
"https://cdn.discordapp.com/",
this.config.ImageProxy
);
}
const describe: MJDescribe = {
id: id,
flags: message.flags,
descriptions: embeds?.[0]?.description.split("\n\n"),
uri: uri,
proxy_url: embeds?.[0]?.image?.proxy_url,
options: formatOptions(components),
};
this.emitMJ(id, describe);
break;
case "prefer remix":
if (content != "") {
this.emit("prefer-remix", content);
}
break;
case "shorten":
const shorten: MJShorten = {
description: embeds?.[0]?.description,
prompts: formatPrompts(embeds?.[0]?.description as string),
options: formatOptions(components),
id,
flags: message.flags,
};
this.emitMJ(id, shorten);
break;
case "info":
this.emit("info", embeds?.[0]?.description);
case "subscribe":
this.emit("subscribe", components?.[0]?.components?.[0]);
return;
}
}
if (embeds?.[0]) {
// message is not able to continue
if (!attachments || !attachments.length || !components || !components.length) {
const { description, title, color } = embeds[0];
switch (color) {
case 16711680: //error
const error = new Error(description);
this.EventError(id, error);
return;
default:
break;
}
if (title === "Duplicate images detected") {
const error = new Error(description);
this.EventError(id, error);
return;
}
}
}
if (content) {
this.processingImage(message);
}
}
//interaction success
private async onInteractionSuccess({
nonce,
id,
}: {
nonce: string;
id: string;
}) {
// this.log("interactionSuccess", nonce, id);
const event = this.getEventByNonce(nonce);
if (!event) {
return;
}
// event.onmodal && event.onmodal(nonce, id);
}
//modal create
private async onModalCreate({
nonce,
id,
custom_id,
components,
}: {
nonce: string;
id: string;
custom_id: string;
components: any;
}) {
this.log("modalCreate", nonce, id);
const event = this.getEventByNonce(nonce);
if (!event) {
return;
}
const prompt_custom_id = components[0]?.components[0]?.custom_id;
event.onmodal && event.onmodal(nonce, id, custom_id, prompt_custom_id);
}
private async onReady(user: any) {
this.UserId = user.id;
}
private async onMessageCreate(message: any) {
const { channel_id, author, interaction } = message;
if (channel_id !== this.config.ChannelId) return;
if (author?.id !== this.config.BotId) return;
if (interaction && interaction.user.id !== this.UserId) return;
// this.log("[messageCreate]", JSON.stringify(message));
this.messageCreate(message);
}
private async onMessageUpdate(message: any) {
const { channel_id, author, interaction } = message;
if (channel_id !== this.config.ChannelId) return;
if (author?.id !== this.config.BotId) return;
if (interaction && interaction.user.id !== this.UserId) return;
// this.log("[messageUpdate]", JSON.stringify(message));
this.messageUpdate(message);
}
private async onMessageDelete(message: any) {
const { channel_id, id } = message;
if (channel_id !== this.config.ChannelId) return;
// @ts-ignore
for (const [key, value] of this.waitMjEvents.entries()) {
if (value.id === id) {
this.waitMjEvents.set(key, { ...value, del: true });
}
}
}
// parse message from ws
private parseMessage(msg: any) {
const operate = msg.op as number;
const type = msg.t as string
if (!type) {
this.log("event", JSON.stringify(msg));
return;
} else {
this.log("event", type);
}
if (!isNaN(msg.s)) {
this.lastSequence = msg.s;
}
// if (operate === 10 && msg?.d?.heartbeat_interval) {
// this.heartbeat(msg.d.heartbeat_interval!);
// }
const message = msg.d;
if (message.channel_id === this.config.ChannelId) {
this.log("message", JSON.stringify(msg));
} else if (!type.includes("MESSAGE_") && type !== 'READY') {
this.log("operations", msg.t);
}
switch (type) {
case "READY":
if (message.session_id) {
this.config.ActiveSessionId = message.session_id;
}
this.emitSystem("ready", message.user);
break;
case "INTERACTION_IFRAME_MODAL_CREATE":
if (message.nonce) {
this.emit(message.nonce, message);
}
break;
case "INTERACTION_MODAL_CREATE":
if (message.nonce) {
this.emitSystem("modalCreate", message);
}
break;
case "MESSAGE_CREATE":
this.emitSystem("messageCreate", message);
break;
case "MESSAGE_UPDATE":
this.emitSystem("messageUpdate", message);
break;
case "MESSAGE_DELETE":
this.emitSystem("messageDelete", message);
break;
case "INTERACTION_SUCCESS":
this.emitSystem("interactionSuccess", message);
break;
case "INTERACTION_CREATE":
if (message.nonce) {
this.emitSystem("interactionCreate", message);
}
break;
default:
break;
}
}
//continue click appeal or Acknowledged
private async continue(message: any) {
const { components, id, flags, nonce } = message;
const appeal = components[0]?.components[0];
this.log("appeal", appeal);
if (appeal) {
var newnonce = nextNonce();
const httpStatus = await this.MJApi.CustomApi({
msgId: id,
customId: appeal.custom_id,
flags,
nonce: newnonce,
});
this.log("appeal.httpStatus", httpStatus);
if (httpStatus == 204) {
//todo
this.log("new nonce", newnonce);
this.on(newnonce, (data) => {
this.log("nonce data", data);
this.emit(nonce, data);
});
}
}
}
private async verifyHuman(message: any) {
const { HuggingFaceToken } = this.config;
if (HuggingFaceToken === "" || !HuggingFaceToken) {
this.log("HuggingFaceToken is empty");
return;
}
const { embeds, components, id, flags, nonce } = message;
const uri = embeds[0].image.url;
const categories = components[0].components;
const classify = categories.map((c: any) => c.label);
const verifyClient = new VerifyHuman(this.config);
const category = await verifyClient.verify(uri, classify);
if (category) {
const custom_id = categories.find(
(c: any) => c.label === category
).custom_id;
var newnonce = nextNonce();
const httpStatus = await this.MJApi.CustomApi({
msgId: id,
customId: custom_id,
flags,
nonce: newnonce,
});
if (httpStatus == 204) {
this.on(newnonce, (data) => {
this.emit(nonce, data);
});
}
this.log("verifyHumanApi", httpStatus, custom_id, message.id);
}
}
private EventError(id: string, error: Error) {
const event = this.getEventById(id);
if (!event) {
return;
}
const eventMsg: MJEmit = {
error,
};
this.log("MJ ERROR:", error.message);
this.emit(event.nonce, eventMsg);
}
private done(message: any) {
const { content, id, attachments, components, flags } = message;
const { url, proxy_url, width, height } = attachments[0];
let uri = url;
if (this.config.ImageProxy !== "") {
uri = uri.replace("https://cdn.discordapp.com/", this.config.ImageProxy);
}
let hash: string | undefined;
if (components && components.length) {
hash = componentsToHash(components);
}
if (!hash) {
hash = uriToHash(url);
}
const MJmsg: MJMessage = {
id,
flags,
content,
hash: uriToHash(url),
progress: "done",
uri,
proxy_url,
options: formatOptions(components),
width,
height,
};
this.filterMessages(MJmsg);
return;
}
private processingImage(message: any, isJobQueued?: boolean) {
const { content, id, attachments, flags, components } = message;
let event: WaitMjEvent | undefined;
let hash: string | undefined;
let uri, proxy_url, width, height;
if (components && components.length) {
hash = componentsToHash(components);
}
if (attachments && attachments.length) {
const { url, filename } = attachments[0];
uri = url;
proxy_url = attachments[0].proxy_url;
width = attachments[0].width;
height = attachments[0].height;
if (this.config.ImageProxy !== "") {
uri = uri.replace("https://cdn.discordapp.com/", this.config.ImageProxy);
}
if (!hash) {
hash = filenameToHash(filename);
}
}
if (hash) {
// found same hash event
event = this.getEventByHash(hash);
this.log('find event by hash', event);
}
// track event by msg id for imagine at the begining.
if (!event) {
event = this.getEventById(id);
this.log('find old event by id', event);
}
if (event && content) {
event.prompt = content;
// update hash for the first time, in imagine case
if (hash && !event.hash) {
this.log('update event by hash', event.id, hash);
this.updateHashByid(id, hash);
event.hash = hash;
}
}
// while MJ queued, this will tell the status
const progress = isJobQueued ? 'Job queued' : content2progress(content);
const MJmsg: MJMessage = {
uri: uri,
proxy_url: proxy_url,
content: content,
flags: flags,
hash,
options: components && components.length && formatOptions(components),
progress,
width,
height,
};
const eventMsg: MJEmit = {
message: MJmsg,
};
if (event) {
this.emitImage(event.nonce, eventMsg);
} else {
this.emitImage('OTHER_MSG', eventMsg);
}
}
private async filterMessages(MJmsg: MJMessage) {
// delay 300ms for discord message delete
await this.timeout(300);
let event: WaitMjEvent | undefined;
if (MJmsg.hash) {
event = this.getEventByHash(MJmsg.hash);
}
// somecase MJ throw finished image directly.
if (!event) {
event = this.getEventByContent(MJmsg.content);
}
const eventMsg: MJEmit = {
message: MJmsg,
};
if (!event) {
this.log("FilterMessages not found", MJmsg.hash, this.waitMjEvents);
this.emitImage('OTHER_MSG', eventMsg);
return;
}
this.emitImage(event.nonce, eventMsg);
}
private getEventByContent(content: string) {
const prompt = content2prompt(content);
//fist del message
// @ts-ignore
for (const [key, value] of this.waitMjEvents.entries()) {
const prompCache = content2prompt(value?.prompt as string);
if (
value.del === true &&
prompCache.includes(prompt)
) {
return value as WaitMjEvent;
}
}
// @ts-ignore
for (const [key, value] of this.waitMjEvents.entries()) {
const prompCache = content2prompt(value?.prompt as string);
if (prompCache.includes(prompt)) {
return value as WaitMjEvent;
}
}
}
private getEventByHash(hash: string) {
// @ts-ignore
for (const [key, value] of this.waitMjEvents.entries()) {
if (value.hash === hash) {
return value as WaitMjEvent;
}
}
}
private getEventById(id: string) {
// @ts-ignore
for (const [key, value] of this.waitMjEvents.entries()) {
if (value.id === id) {
return value as WaitMjEvent;
}
}
}
private getEventByNonce(nonce: string) {
// @ts-ignore
for (const [key, value] of this.waitMjEvents.entries()) {
if (value.nonce === nonce) {
return value as WaitMjEvent;
}
}
}
private getNonceFromContent(content: string) {
return content.match(/\[(.*?)\]/)?.[1];
}
private updateMjEventIdByNonce(nonce: string, id: string) {
if (nonce === "" || id === "") return;
let event = this.waitMjEvents.get(nonce);
if (!event) return;
event.id = id;
this.log("updateMjEventIdByNonce success", event);
return event;
}
private updateHashByNonce(nonce: string, hash: string) {
if (nonce === "" || hash === "") return;
let event = this.waitMjEvents.get(nonce);
if (!event) return;
event.hash = hash;
this.log("updateHashByNonce success", event);
return event;
}
private updateHashByid(id: string, hash: string) {
if (id === "" || hash === "") return;
let event = this.getEventById(id);
if (!event) return;
event.hash = hash;
this.log("updateHashByMjEventId success", event);
return event;
}
protected async log(...args: any[]) {
this.config.Debug && console.info(...args, new Date().toISOString());
}
emit(event: string, message: any) {
this.event
.filter((e) => e.event === event)
.forEach((e) => e.callback(message));
}
private emitImage(type: string, message: MJEmit) {
this.emit(type, message);
}
//FIXME: emitMJ rename
private emitMJ(id: string, data: any) {
const event = this.getEventById(id);
if (!event) return;
this.emit(event.nonce, data);
}
on(event: string, callback: (message: any) => void) {
this.event.push({ event, callback });
}
onSystem(
event:
| "ready"
| "messageCreate"
| "messageUpdate"
| "messageDelete"
| "modalCreate"
| "interactionCreate"
| "interactionSuccess",
callback: (message: any) => void
) {
this.on(event, callback);
}
private emitSystem(
type:
| "ready"
| "messageCreate"
| "messageUpdate"
| "messageDelete"
| "modalCreate"
| "interactionSuccess"
| "interactionCreate",
message: MJEmit
) {
this.emit(type, message);
}
once(event: string, callback: (message: any) => void) {
const once = (message: any) => {
this.remove(event, once);
callback(message);
};
this.event.push({ event, callback: once });
}
remove(event: string, callback: (message: any) => void) {
this.event = this.event.filter(
(e) => e.event !== event && e.callback !== callback
);
}
removeEvent(event: string) {
this.event = this.event.filter((e) => e.event !== event);
}
onceMJ(nonce: string, callback: (data: any) => void) {
const once = (message: any) => {
this.remove(nonce, once);
//FIXME: removeWaitMjEvent
this.removeWaitMjEvent(nonce);
callback(message);
};
//FIXME: addWaitMjEvent
this.waitMjEvents.set(nonce, { nonce });
this.event.push({ event: nonce, callback: once });
}
private removeSkipMessageId(messageId: string) {
const index = this.skipMessageId.findIndex((id) => id !== messageId);
if (index !== -1) {
this.skipMessageId.splice(index, 1);
}
}
private removeWaitMjEvent(nonce: string) {
this.waitMjEvents.delete(nonce);
}
onceImage(nonce: string, callback: (data: MJEmit) => void) {
const once = (data: MJEmit) => {
const { message, error } = data;
if (error || (message && message.progress === "done")) {
this.remove(nonce, once);
}
callback(data);
};
this.event.push({ event: nonce, callback: once });
}
async waitImageMessage({
nonce,
prompt,
onmodal,
messageId,
loading,
}: {
nonce: string;
prompt?: string;
messageId?: string;
onmodal?: OnModal;
loading?: LoadingHandler;
}) {
if (messageId) this.skipMessageId.push(messageId);
return new Promise<MJMessage | null>((resolve, reject) => {
const handleImageMessage = ({ message, error }: MJEmit) => {
if (error) {
this.removeWaitMjEvent(nonce);
reject(error);
return;
}
if (message && message.progress === "done") {
this.removeWaitMjEvent(nonce);
messageId && this.removeSkipMessageId(messageId);
resolve(message);
return;
}
message && loading && loading({
uri: message.uri,
progress: message.progress || "",
options: message.options,
hash: message.hash,
});
};
this.waitMjEvents.set(nonce, {
nonce,
prompt,
onmodal: async (oldnonce, id, custom_id, prompt_custom_id) => {
if (onmodal === undefined) {
// reject(new Error("onmodal is not defined"))
return "";
}
var nonce = await onmodal(oldnonce, id, custom_id, prompt_custom_id);
if (nonce === "") {
// reject(new Error("onmodal return empty nonce"))
return "";
}
this.removeWaitMjEvent(oldnonce);
this.waitMjEvents.set(nonce, { nonce });
this.onceImage(nonce, handleImageMessage);
return nonce;
},
});
this.onceImage(nonce, handleImageMessage);
});
}
async waitOnceMJ(nonce: string) {
return new Promise<any>((resolve) => {
this.onceMJ(nonce, (message) => {
resolve(message);
});
});
}
async waitOnce(nonce: string) {
return new Promise<any>((resolve) => {
this.once(nonce, (message) => {
resolve(message);
});
});
}
}