@salutejs/jazz-sdk-web-plugins
Version:
Jazz SDK plugins
302 lines (301 loc) • 12 kB
JavaScript
const inWorker = typeof document === 'undefined';
// FIXME: duplication
const W = 1280;
const H = 720;
export function initializeBackend(application, port, filepaths) {
function send_text(channel, text) {
port.postMessage({ cmd: 'text', channel, text });
}
class OrbbieglFacade {
version;
Orbbiegl;
session;
jazzGraph;
screenGraph;
wallpaperSrc;
constructor(Orbbiegl) {
this.Orbbiegl = Orbbiegl;
if (Orbbiegl.Session === undefined) {
throw new Error('Orbbiegl module was not loaded properly');
}
this.session = new Orbbiegl.Session(true);
if (Orbbiegl.SberJazzGraph !== undefined) {
this.jazzGraph = new Orbbiegl.SberJazzGraph();
}
if (Orbbiegl.SmartScreenGraph !== undefined) {
this.screenGraph = new Orbbiegl.SmartScreenGraph(0.5, 0.5, 0.7, 0.5);
}
this.version = this.session.version;
}
reset() {
this.jazzGraph?.reset();
this.screenGraph?.reset();
}
setWallpaper(img) {
this.wallpaperSrc?.close();
this.wallpaperSrc = img;
}
updateWallpaper() {
if (this.wallpaperSrc !== undefined) {
const w = this.wallpaperSrc.width;
const h = this.wallpaperSrc.height;
this.jazzGraph?.setWallpaper(this.wallpaperSrc, w, h);
this.wallpaperSrc = undefined;
}
}
process_video(frame, w, h) {
const t0 = performance.now();
this.updateWallpaper();
this.jazzGraph?.process(frame, w, h);
return performance.now() - t0;
}
process_capture(frame, w, h) {
const t0 = performance.now();
const result = this.screenGraph?.process(frame, w, h);
const items = [];
if (result) {
for (let i = 0; i < result.size(); ++i) {
const obj = result.get(i);
items.push({
x: obj.x,
y: obj.y,
width: obj.width,
height: obj.height,
confidence: obj.confidence,
code: obj.code,
text: obj.text,
format: obj.format,
});
}
result.delete();
}
return { items, time: performance.now() - t0 };
}
process_diag(frame, w, h) {
if (this.jazzGraph && this.screenGraph) {
const buffer1 = this.jazzGraph.diagnostics(frame, w, h);
const arr1 = buffer1.asInt8Array();
const buffer2 = this.screenGraph.diagnostics(frame, w, h);
const arr2 = buffer2.asInt8Array();
const blob = new Blob([arr1, arr2], {
type: 'application/octet-stream',
});
buffer1.delete();
buffer2.delete();
return blob;
}
}
warmUp() {
this.jazzGraph?.warmUp();
this.screenGraph?.warmUp();
return new Promise((resolve) => {
const checkWarmUp = () => setTimeout(() => {
if (this.session && this.session.warmUp() < 1.0) {
resolve();
}
else {
checkWarmUp();
}
}, 100);
checkWarmUp();
});
}
destroy() {
this.screenGraph?.delete();
this.jazzGraph?.delete();
this.session?.delete();
}
}
function buildModule(filepaths) {
return {
// OffscreenCanvas is canvas that used inside worker, where `document` is not present.
canvas: inWorker
? new OffscreenCanvas(W, H)
: document.createElement('canvas'),
application,
facade: undefined,
needWarmUp: false,
bg_modes: undefined,
bt_modes: undefined,
locateFile: (fileName) => {
const map = {
'orbbiegl-jazz.wasm': filepaths.wasmFile,
'orbbiegl-jazz.data': filepaths.dataFile,
};
if (map[fileName] === undefined) {
// Return original path in case of known, but undefined value.
if (Object.keys(map).includes(fileName)) {
return fileName;
}
// Throw error otherwise.
throw new Error(`Unexpected static file for Orbbiegl: ${fileName}`);
}
return map[fileName];
},
print: function (text) {
send_text('log', text);
},
printErr: function (text) {
send_text('error', text);
},
progress: function (value) {
if (this.facade === undefined) {
port.postMessage({ cmd: 'notReady', progress: value });
}
},
onRuntimeInitialized: function () {
if (this.BackgroundMode) {
this.bg_modes = {
Blur: this.BackgroundMode.Blur,
Wallpaper: this.BackgroundMode.Wallpaper,
Gray: this.BackgroundMode.Gray,
Sepia: this.BackgroundMode.Sepia,
GreenScreenWallpaper: this.BackgroundMode.GreenScreenWallpaper,
None: this.BackgroundMode.None,
};
}
if (this.BeautificationMode) {
this.bt_modes = {
OFF: this.BeautificationMode.OFF,
ON: this.BeautificationMode.ON,
Debug: this.BeautificationMode.Debug,
Max: this.BeautificationMode.Max,
};
}
const facade = new OrbbieglFacade(this);
if (this.needWarmUp) {
facade.warmUp();
}
this.facade = facade;
port.postMessage({
cmd: 'ready',
application: this.application,
version: facade.version,
});
},
destroy: function () {
this.facade?.destroy();
this.facade = undefined;
},
};
}
let Orbbiegl;
if (!inWorker && filepaths !== undefined) {
Orbbiegl = buildModule(filepaths);
}
port.onmessage = function (evt) {
const data = evt.data;
const cmd = data.cmd;
if (cmd == 'init') {
const { needWarmUp, filepaths = {} } = data;
if (Orbbiegl === undefined) {
Orbbiegl = buildModule(filepaths);
}
if (Orbbiegl.facade) {
port.postMessage({ cmd: 'ready', application: application });
return;
}
if (needWarmUp !== undefined) {
Orbbiegl.needWarmUp = data.needWarmUp;
}
const importPath = filepaths.jsFile ?? './orbbiegl-jazz.js?asset';
import(`${importPath}`).then((mod) => mod.default(Orbbiegl));
return;
}
const assert = (orbbiegl) => {
if (orbbiegl.facade !== undefined &&
orbbiegl.bg_modes !== undefined &&
orbbiegl.bt_modes !== undefined &&
orbbiegl.Session !== undefined) {
return orbbiegl;
}
return false;
};
const orb = assert(Orbbiegl);
if (orb) {
switch (cmd) {
case 'wallpaper':
orb.facade.setWallpaper(data.image);
break;
case 'mode': {
const mode = data.bg_mode;
if (mode !== undefined) {
orb.facade.jazzGraph?.setBackgroundMode(orb.bg_modes[mode]);
}
if (data.awb_enabled !== undefined) {
orb.facade.jazzGraph?.setAwbEnabled(data.awb_enabled);
}
if (data.agc_enabled !== undefined) {
orb.facade.jazzGraph?.setAgcEnabled(data.agc_enabled);
}
if (data.sf_enabled !== undefined) {
orb.facade.jazzGraph?.setSFEnabled(data.sf_enabled);
}
if (data.bt_mode !== undefined) {
orb.facade.jazzGraph?.setBeautificationMode(orb.bt_modes[data.bt_mode]);
}
break;
}
case 'frame': {
const dimensions = (frame) => {
if (frame.constructor.name === 'VideoFrame') {
const f = frame;
return [f.codedWidth, f.codedHeight];
}
else {
const f = frame;
return [f.width, f.height];
}
};
const { frame, reset, klass } = data;
const [w, h] = dimensions(frame);
if (klass == 'diag') {
const result = orb.facade.process_diag(frame, w, h);
if (result) {
const cs = new CompressionStream('gzip');
const rs = result.stream().pipeThrough(cs);
new Response(rs)
.blob()
.then((blob) => port.postMessage({ cmd: 'result', result: { blob } }));
}
}
else if (klass == 'video') {
if (reset) {
orb.facade.jazzGraph?.reset();
}
const time = orb.facade.process_video(frame, w, h);
// OffscreenCanvas has this method, HTMLCanvas not.
const result = orb.canvas.transferToImageBitmap !==
undefined
? orb.canvas.transferToImageBitmap()
: orb.canvas;
port.postMessage({ cmd: 'result', result: { frame: result, time } }, [result]); // FIXME: hack, HTMLCanvasElement is not transferable, but we it will be a DummyPort anyway
}
else if (klass == 'capture') {
const items = orb.facade.process_capture(frame, w, h);
const result = orb.canvas.transferToImageBitmap !==
undefined
? orb.canvas.transferToImageBitmap()
: orb.canvas;
port.postMessage({ cmd: 'result', result: { frame: result, ...items } }, [result]);
}
frame.close?.();
break;
}
case 'close':
Orbbiegl.destroy();
port.postMessage({ cmd: 'closed' });
port.close?.();
break;
default:
break;
}
}
else {
Orbbiegl.printErr('Orbbiegl is not initialized');
}
};
}
if (inWorker) {
initializeBackend(name, self);
}