romaine
Version:
React OpenCV Manipulation and Image Narration & Editing
1,333 lines (1,309 loc) • 85.2 kB
JSX
import { createContext, useState, useRef, useCallback, useMemo, useEffect, useReducer, useContext, forwardRef, useImperativeHandle } from 'react';
import { jsx, jsxs, Fragment } from 'react/jsx-runtime';
import T from 'prop-types';
import Draggable from 'react-draggable';
const calcDims = (width, height, externalMaxWidth, externalMaxHeight) => {
const ratio = width / height;
const maxWidth = externalMaxWidth || window.innerWidth;
const maxHeight = externalMaxHeight || window.innerHeight;
const calculated = {
width: maxWidth,
height: Math.round(maxWidth / ratio),
ratio: ratio,
};
if (calculated.height > maxHeight) {
calculated.height = maxHeight;
calculated.width = Math.round(maxHeight * ratio);
}
return calculated;
};
function isCrossOriginURL(url) {
const { location } = window;
const parts = url.match(/^(\w+:)\/\/([^:/?#]*):?(\d*)/i);
return (parts !== null &&
(parts[1] !== location.protocol ||
parts[2] !== location.hostname ||
parts[3] !== location.port));
}
/**
* Takes a file or a string, loads it and turns it into a DataURL
* @param file as File | string
* @returns DataURL (as string)
*/
const readFile = (file) => new Promise((resolve, reject) => {
if (file instanceof File) {
const reader = new FileReader();
reader.onload = () => {
resolve(reader.result);
};
reader.onerror = (err) => {
reject(err);
};
reader.readAsDataURL(file);
}
else if (typeof file === "string")
return resolve(file);
else
reject();
});
// https://amin-ahmadi.com/2016/03/24/sepia-filter-opencv/
const applyFilter = async (cv, docCanvas, filterCvParams = {}) => {
// default options
const options = {
blur: false,
th: true,
thMode: cv.ADAPTIVE_THRESH_MEAN_C,
thMeanCorrection: 15,
thBlockSize: 25,
thMax: 255,
grayScale: true,
...filterCvParams,
};
const dst = cv.imread(docCanvas);
if (options.grayScale) {
cv.cvtColor(dst, dst, cv.COLOR_RGBA2GRAY, 0);
}
if (options.blur) {
const ksize = new cv.Size(5, 5);
cv.GaussianBlur(dst, dst, ksize, 0, 0, cv.BORDER_DEFAULT);
}
if (options.th) {
if (options.grayScale) {
cv.adaptiveThreshold(dst, dst, options.thMax, options.thMode, cv.THRESH_BINARY, options.thBlockSize, options.thMeanCorrection);
}
else {
//@ts-ignore need to fix this type error (add to OpenCV type)
dst.convertTo(dst, -1, 1, 60);
cv.threshold(dst, dst, 170, 255, cv.THRESH_BINARY);
}
}
cv.imshow(docCanvas, dst);
};
/**
* perspective cropping utility (AKA keystone correction)
* @param cv openCv
* @param src The source image pointer
* @param cropPoints
* @param imageResizeRatio
*/
const warpPerspective = (cv, src, cropPoints, imageResizeRatio) => {
// const src = cv.imread(docCanvas);
const bR = cropPoints["right-bottom"];
const bL = cropPoints["left-bottom"];
const tR = cropPoints["right-top"];
const tL = cropPoints["left-top"];
// create source coordinates matrix
const sourceCoordinates = [tL, tR, bR, bL].map((point) => [
point.x / imageResizeRatio,
point.y / imageResizeRatio,
]);
// get max width
const maxWidth = Math.max(bR.x - bL.x, tR.x - tL.x) / imageResizeRatio;
// get max height
const maxHeight = Math.max(bL.y - tL.y, bR.y - tR.y) / imageResizeRatio;
// create dest coordinates matrix
const destCoordinates = [
[0, 0],
[maxWidth - 1, 0],
[maxWidth - 1, maxHeight - 1],
[0, maxHeight - 1],
];
// convert to open cv matrix objects
const Ms = cv.matFromArray(4, 1, cv.CV_32FC2, [].concat(...sourceCoordinates));
const Md = cv.matFromArray(4, 1, cv.CV_32FC2, [].concat(...destCoordinates));
const transformMatrix = cv.getPerspectiveTransform(Ms, Md);
// set new image size
const dsize = new cv.Size(maxWidth, maxHeight);
// perform warp
cv.warpPerspective(src, src, transformMatrix, dsize, cv.INTER_LINEAR, cv.BORDER_CONSTANT, new cv.Scalar());
// cv.imshow(docCanvas, src);
// src.delete();
Ms.delete();
Md.delete();
transformMatrix.delete();
};
function multiplyMatrices(m1, m2) {
const result = [];
for (let i = 0; i < m1.length; i++) {
result[i] = [];
for (let j = 0; j < m2[0].length; j++) {
let sum = 0;
for (let k = 0; k < m1[0].length; k++) {
sum += m1[i][k] * m2[k][j];
}
result[i][j] = sum;
}
}
return result;
}
/**
* Function that adds transparent (or whatever the fill color is) to the image canvas
* @see https://stackoverflow.com/questions/43391205/add-padding-to-images-to-get-them-into-the-same-shape#43391469
* @example
* addPadding(cv, canvasRef.current, { left: 500 }, { setPreviewPaneDimensions, showPreview });
*/
const addPadding = (cv, src, { top = 0, bottom = 0, left = 0, right = 0 }, { showPreview, setPreviewPaneDimensions }) => {
const dst = new cv.Mat();
// @ts-ignore
cv.copyMakeBorder(cv.imread(src), dst, top, bottom, left, right, cv.BORDER_CONSTANT);
setPreviewPaneDimensions({
width: src.width + left + right,
height: src.height + top + bottom,
});
cv.imshow(src, dst);
showPreview();
dst.delete();
};
/**
* perspective cropping utility (AKA keystone correction)
* @param cv openCv
* @param docCanvas
* @param cropPoints
* @param imageResizeRatio
* @param setPreviewPaneDimensions
*/
const cropOpenCV = (cv, dst, cropPoints, imageResizeRatio
// _canvasRef: HTMLCanvasElement,
) => {
// const dst = cv.imread(docCanvas);
// const bR = cropPoints["right-bottom"];
const bL = cropPoints["left-bottom"];
const tR = cropPoints["right-top"];
const tL = cropPoints["left-top"];
const l = tL.x / imageResizeRatio;
const t = tL.y / imageResizeRatio;
const r = tR.x / imageResizeRatio;
const b = bL.y / imageResizeRatio;
const M = cv.matFromArray(2, 3, cv.CV_64FC1, [1, 0, -l, 0, 1, -t]);
// let dsize = new cv.Size(dst.rows - t - b, dst.cols - l - r);
// const dsize = new cv.Size(dst.rows - l - r, dst.cols - t - b);
const dsize = new cv.Size(dst.cols - l - (dst.cols - r), dst.rows - t - (dst.rows - b));
cv.warpAffine(dst, dst, M, dsize, cv.INTER_LINEAR, cv.BORDER_CONSTANT, new cv.Scalar());
// setTimeout(async () => {
// cv.imshow(canvasRef, dst);
// }, 0);
// dst.delete();
M.delete();
};
const MAX_GRABCUT_DIM = 800;
// --- Module-level GrabCut state storage ---
let storedMask = null;
let storedBgdModel = null;
let storedFgdModel = null;
let storedScaleFactor = 1;
let storedSmallW = 0;
let storedSmallH = 0;
let lastAppliedStrokes = [];
const clearGrabCutState = () => {
try {
storedMask?.delete();
}
catch { /* noop */ }
try {
storedBgdModel?.delete();
}
catch { /* noop */ }
try {
storedFgdModel?.delete();
}
catch { /* noop */ }
storedMask = null;
storedBgdModel = null;
storedFgdModel = null;
storedScaleFactor = 1;
storedSmallW = 0;
storedSmallH = 0;
lastAppliedStrokes = [];
};
const hasGrabCutMask = () => storedMask !== null;
const getGrabCutScaleFactor = () => storedScaleFactor;
const setLastAppliedStrokes = (s) => {
lastAppliedStrokes = s;
};
const getLastAppliedStrokes = () => lastAppliedStrokes;
/**
* Shared pipeline: convert raw GrabCut mask to binary, apply morphology,
* blur, upscale, and write to src alpha channel.
*/
const applyMaskToAlpha = (cv, canvas, src, rawMask, origW, origH, scaleFactor) => {
const mats = [];
const del = (m) => { mats.push(m); return m; };
try {
// Clone so we don't modify the stored mask
const binMask = del(rawMask.clone());
const maskData = binMask.data;
for (let i = 0; i < maskData.length; i++) {
maskData[i] = (maskData[i] === 1 || maskData[i] === 3) ? 255 : 0;
}
// Morphological cleanup if available
const hasMorphology = typeof cv.getStructuringElement === "function" &&
typeof cv.morphologyEx === "function";
if (hasMorphology) {
const kernel = del(cv.getStructuringElement(cv.MORPH_ELLIPSE, new cv.Size(5, 5)));
cv.morphologyEx(binMask, binMask, cv.MORPH_CLOSE, kernel);
cv.morphologyEx(binMask, binMask, cv.MORPH_OPEN, kernel);
}
// Edge feathering
const blurred = del(new cv.Mat());
cv.GaussianBlur(binMask, blurred, new cv.Size(5, 5), 0, 0, cv.BORDER_DEFAULT);
// Upscale mask back to original dimensions
let finalMask;
if (scaleFactor < 1) {
finalMask = del(new cv.Mat());
cv.resize(blurred, finalMask, new cv.Size(origW, origH), 0, 0, cv.INTER_LINEAR);
}
else {
finalMask = blurred;
}
// Apply mask as alpha channel
const srcData = src.data;
const finalMaskData = finalMask.data;
const totalPixels = origW * origH;
for (let i = 0; i < totalPixels; i++) {
srcData[i * 4 + 3] = finalMaskData[i];
}
cv.imshow(canvas, src);
}
finally {
for (const m of mats) {
try {
m.delete();
}
catch { /* noop */ }
}
}
};
/**
* Remove the background from an image using OpenCV's GrabCut algorithm.
* Stores the GrabCut mask for subsequent refinement via refineBackground().
*/
const removeBackground = (cv, canvas, src) => {
if (!cv.grabCut) {
throw new Error("cv.grabCut is not available in this OpenCV build");
}
// Clear any previous stored state
clearGrabCutState();
const origW = src.cols;
const origH = src.rows;
const maxDim = Math.max(origW, origH);
const sf = maxDim > MAX_GRABCUT_DIM ? MAX_GRABCUT_DIM / maxDim : 1;
const sW = Math.round(origW * sf);
const sH = Math.round(origH * sf);
const mats = [];
const del = (m) => { mats.push(m); return m; };
try {
// Downscale if needed
let small;
if (sf < 1) {
small = del(new cv.Mat());
cv.resize(src, small, new cv.Size(sW, sH), 0, 0, cv.INTER_AREA);
}
else {
small = del(src.clone());
}
const rgb = del(new cv.Mat());
cv.cvtColor(small, rgb, cv.COLOR_RGBA2RGB, 0);
const mask = del(new cv.Mat());
const bgdModel = del(new cv.Mat());
const fgdModel = del(new cv.Mat());
const insetX = Math.max(1, Math.floor(sW * 0.03));
const insetY = Math.max(1, Math.floor(sH * 0.03));
const rect = new cv.Rect(insetX, insetY, sW - insetX * 2, sH - insetY * 2);
// Pass 1: rectangle init
cv.grabCut(rgb, mask, rect, bgdModel, fgdModel, 5, cv.GC_INIT_WITH_RECT);
// Pass 2: mask refinement
cv.grabCut(rgb, mask, rect, bgdModel, fgdModel, 3, cv.GC_INIT_WITH_MASK ?? 1);
// Store mask & models for future refinement (before binary conversion)
storedMask = mask.clone();
storedBgdModel = bgdModel.clone();
storedFgdModel = fgdModel.clone();
storedScaleFactor = sf;
storedSmallW = sW;
storedSmallH = sH;
// Apply the mask to the image
applyMaskToAlpha(cv, canvas, src, mask, origW, origH, sf);
}
finally {
for (const m of mats) {
try {
m.delete();
}
catch { /* noop */ }
}
}
};
/**
* Refine the background removal using user-supplied brush strokes.
* Strokes mark pixels as definite foreground or background, then
* GrabCut re-runs with GC_INIT_WITH_MASK.
*/
const refineBackground = (cv, canvas, src, strokes) => {
if (!storedMask || !storedBgdModel || !storedFgdModel) {
throw new Error("No GrabCut mask available. Run removeBackground first.");
}
if (!strokes || strokes.length === 0)
return;
const origW = src.cols;
const origH = src.rows;
const mats = [];
const del = (m) => { mats.push(m); return m; };
try {
// Downscale current image for GrabCut
let small;
if (storedScaleFactor < 1) {
small = del(new cv.Mat());
cv.resize(src, small, new cv.Size(storedSmallW, storedSmallH), 0, 0, cv.INTER_AREA);
}
else {
small = del(src.clone());
}
const rgb = del(new cv.Mat());
cv.cvtColor(small, rgb, cv.COLOR_RGBA2RGB, 0);
// Clone the stored mask and paint user strokes onto it
const refineMask = del(storedMask.clone());
const maskData = refineMask.data;
for (const stroke of strokes) {
const gcValue = stroke.mode === "fg" ? 1 : 0; // GC_FGD=1, GC_BGD=0
for (const pt of stroke.points) {
const r = stroke.brushRadius;
const rSq = r * r;
const yMin = Math.max(0, pt.y - r);
const yMax = Math.min(storedSmallH - 1, pt.y + r);
const xMin = Math.max(0, pt.x - r);
const xMax = Math.min(storedSmallW - 1, pt.x + r);
for (let my = yMin; my <= yMax; my++) {
const dy = my - pt.y;
for (let mx = xMin; mx <= xMax; mx++) {
const dx = mx - pt.x;
if (dx * dx + dy * dy <= rSq) {
maskData[my * storedSmallW + mx] = gcValue;
}
}
}
}
}
// Re-run GrabCut with user-updated mask
const dummyRect = new cv.Rect(0, 0, 1, 1);
cv.grabCut(rgb, refineMask, dummyRect, storedBgdModel, storedFgdModel, 3, cv.GC_INIT_WITH_MASK ?? 1);
// Update stored mask with refined result
storedMask.delete();
storedMask = refineMask.clone();
// Apply the refined mask to the image
applyMaskToAlpha(cv, canvas, src, refineMask, origW, origH, storedScaleFactor);
}
finally {
for (const m of mats) {
try {
m.delete();
}
catch { /* noop */ }
}
}
};
/**
*
* @param state RomaineState
* @param payload HistoryAction["payload"]
* @todo
* 1) reduce rotation to one command if previous history is also rotation
*/
const history = (state, payload) => {
switch (payload.cmd) {
case "CLEAR": {
// overwrite with empty array
return {
...state,
history: { commands: [], pointer: 0 },
};
}
case "PUSH": {
if (state.mode === "undo" || state.mode === "redo" || state.mode === null)
return { ...state };
const pointer = state.history.pointer;
let newCommands = state.history.commands;
if (pointer < newCommands.length) {
newCommands = state.history.commands.reduce((a, c, i) => (pointer > i ? [...a, c] : a), []);
}
// Use custom payload from plugin if provided, otherwise derive from state
const entry = payload.customPayload !== undefined
? { cmd: state.mode, payload: payload.customPayload }
: getHistoryFromState(state);
return {
...state,
history: {
...state.history,
commands: [...newCommands, entry],
pointer: pointer + 1,
},
};
}
case "UNDO": {
const pointer = state.history.pointer;
return {
...state,
history: { ...state.history, pointer: pointer ? pointer - 1 : 0 },
};
}
case "REDO": {
const pointer = state.history.pointer;
return {
...state,
history: { ...state.history, pointer: pointer ? pointer - 1 : 0 },
};
}
default:
return { ...state };
}
};
const getHistoryFromState = ({ mode, angle, scale, cropPoints, }) => {
switch (mode) {
case "rotate-left":
return { cmd: mode, payload: angle % 360 };
case "rotate-right":
return { cmd: mode, payload: (360 - angle) % 360 };
case "flip-horizontal":
return { cmd: mode, payload: null };
case "flip-vertical":
return { cmd: mode, payload: null };
case "crop":
return { cmd: mode, payload: cropPoints };
case "perspective-crop":
return { cmd: mode, payload: cropPoints };
case "scale":
return { cmd: mode, payload: scale };
case "remove-background":
return { cmd: mode, payload: null };
case "refine-background":
return { cmd: mode, payload: getLastAppliedStrokes() };
case "full-reset":
throw new Error('error: action "full-reset" should not call `history`.`PUSH`').stack;
case null:
// handles null
throw new Error("error: action of type null should not call `history`.`PUSH`").stack;
default:
// Plugin modes — return generic entry (plugins should use customPayload instead)
return { cmd: mode, payload: null };
}
};
const romaineReducer = (state, action) => {
switch (action.type) {
case "JOIN_PAYLOAD":
return { ...state, ...action.payload };
case "MODE":
if (state.mode === action.payload)
return { ...state, mode: null };
return { ...state, mode: action.payload };
case "ANGLE":
return { ...state, angle: action.payload };
case "SCALE":
return { ...state, scale: action.payload };
case "CROP_POINTS":
if (!action.payload)
return { ...state };
return { ...state, cropPoints: action.payload };
case "HISTORY":
return { ...history(state, action.payload) };
case "IMAGE-UPDATE":
return { ...state, image: action.payload };
default:
return { ...state };
}
};
const initialRomaineState = {
mode: null,
angle: 90,
scale: {
width: 0,
height: 0,
},
image: {
width: 0,
height: 0,
id: null,
},
history: { commands: [], pointer: 0 },
cropPoints: {
"left-top": { x: 0, y: 0 },
"left-bottom": { x: 0, y: 0 },
"right-bottom": { x: 0, y: 0 },
"right-top": { x: 0, y: 0 },
},
};
const moduleConfig = {
wasmBinaryFile: "opencv_js.wasm",
usingWasm: true,
};
const OpenCvContext = createContext({
loaded: false,
romaine: initialRomaineState,
setCropPoints: null,
undo: null,
redo: null,
});
const { Consumer: OpenCvConsumer, Provider } = OpenCvContext;
const scriptId = "openCvScriptTag";
/**
* a romaine context for use in getting openCV and the canvas ref element
* @todo
* 1) Add ref to provider
* 2) See if nonce is really required here
*/
const Romaine = ({ openCvPath, children, onLoad, angle = 90, }) => {
const [loaded, setLoaded] = useState(false);
const [_image, setImage] = useState(null);
const canvasApi = useRef(null);
const handleOnLoad = useCallback(() => {
onLoad && onLoad(window.cv);
setLoaded(true);
}, [onLoad, setLoaded]);
const generateOpenCvScriptTag = useMemo(() => {
// make sure we are in the browser
if (typeof window !== "undefined") {
if (!document.getElementById(scriptId) && !window.cv) {
const js = document.createElement("script");
// append to head
js.id = scriptId;
js.nonce = "8IBTHwOdqNKAWeKl7plt8g==";
js.defer = true;
js.async = true;
js.src = openCvPath || "https://docs.opencv.org/3.4.13/opencv.js";
return js;
}
else if (document.getElementById(scriptId) && !window.cv) {
return document.getElementById(scriptId);
}
}
}, [openCvPath]);
useEffect(() => {
if (window.cv) {
handleOnLoad();
return;
}
// https://docs.opencv.org/3.4/dc/de6/tutorial_js_nodejs.html
// https://medium.com/code-divoire/integrating-opencv-js-with-an-angular-application-20ae11c7e217
// https://stackoverflow.com/questions/56671436/cv-mat-is-not-a-constructor-opencv
moduleConfig.onRuntimeInitialized = handleOnLoad;
window.Module = moduleConfig;
// if (!document.getElementById(scriptId))
if (generateOpenCvScriptTag && !document.getElementById(scriptId))
document.body.appendChild(generateOpenCvScriptTag);
// else handleOnLoad();
}, [openCvPath, handleOnLoad, generateOpenCvScriptTag]);
const [romaine, dispatchRomaine] = useReducer(romaineReducer, initialRomaineState);
const setMode = useCallback((mode) => {
dispatchRomaine({ type: "MODE", payload: mode });
}, [dispatchRomaine]);
const setAngle = useCallback((angle) => {
dispatchRomaine({ type: "ANGLE", payload: angle });
}, [dispatchRomaine]);
const setScale = useCallback((scale) => {
dispatchRomaine({ type: "SCALE", payload: scale });
}, [dispatchRomaine]);
const updateImageInformation = useCallback((image) => {
dispatchRomaine({
type: "SCALE",
payload: {
width: image.width,
height: image.height,
},
});
dispatchRomaine({ type: "IMAGE-UPDATE", payload: image });
}, [dispatchRomaine]);
const { cropPoints, history: { pointer }, } = romaine;
const setCropPoints = useCallback((payload) => {
if (typeof payload === "function")
payload = payload(cropPoints);
dispatchRomaine({ type: "CROP_POINTS", payload });
}, [dispatchRomaine, cropPoints]);
const pushHistory = useCallback((customPayload) => {
dispatchRomaine({ type: "HISTORY", payload: { cmd: "PUSH", customPayload } });
}, [dispatchRomaine]);
const clearHistory = useCallback(() => {
dispatchRomaine({ type: "HISTORY", payload: { cmd: "CLEAR" } });
}, [dispatchRomaine]);
const moveHistory = useCallback((direction) => {
if (direction)
return () => {
dispatchRomaine({ type: "MODE", payload: null });
dispatchRomaine({ type: "HISTORY", payload: { cmd: "UNDO" } });
};
else
return () => {
dispatchRomaine({ type: "MODE", payload: null });
dispatchRomaine({ type: "HISTORY", payload: { cmd: "REDO" } });
};
}, [dispatchRomaine]);
useEffect(() => {
setAngle(angle);
}, [angle, setAngle]);
const memoizedProviderValue = useMemo(() => ({
loaded,
cv: typeof window !== "undefined"
? window?.cv
: null,
romaine: { ...romaine, history: { ...romaine.history }, clearHistory },
canvasApi,
setImage,
setMode,
setAngle,
setScale,
updateImageInformation,
pushHistory,
setCropPoints,
undo: moveHistory(true),
redo: moveHistory(false),
}), [
loaded,
romaine,
pointer,
setMode,
setAngle,
setScale,
pushHistory,
moveHistory,
clearHistory,
setCropPoints,
updateImageInformation,
]);
// const { canvasRef } = useCanvas({ image });
// usePreview({ cv: memoizedProviderValue, canvasRef });
return jsx(Provider, { value: memoizedProviderValue, children: children });
};
Romaine.propTypes = {
openCvPath: T.string,
children: T.node,
onLoad: T.func,
angle: T.number,
};
const useRomaine = () => useContext(OpenCvContext);
/**
* Add default styles to the points
* Makes sure position absolute cannot be overwritten
* @param cropPointStyles
*/
const buildCropPointStyle = (cropPointStyles = {}) => ({
backgroundColor: "transparent",
border: "4px solid #3cabe2",
zIndex: 1001,
borderRadius: "100%",
cursor: "grab",
...cropPointStyles,
position: "absolute",
});
const MAGNIFIER_SIZE = 120;
const ZOOM_LEVEL = 3;
const MAGNIFIER_OFFSET = 30;
const drawMagnifier = (magnifierCanvas, previewCanvas, pointX, pointY) => {
const ctx = magnifierCanvas.getContext("2d");
if (!ctx)
return;
ctx.clearRect(0, 0, MAGNIFIER_SIZE, MAGNIFIER_SIZE);
// Circular clip
ctx.save();
ctx.beginPath();
ctx.arc(MAGNIFIER_SIZE / 2, MAGNIFIER_SIZE / 2, MAGNIFIER_SIZE / 2, 0, Math.PI * 2);
ctx.clip();
// Draw zoomed region from preview canvas
const sourceSize = MAGNIFIER_SIZE / ZOOM_LEVEL;
ctx.drawImage(previewCanvas, pointX - sourceSize / 2, pointY - sourceSize / 2, sourceSize, sourceSize, 0, 0, MAGNIFIER_SIZE, MAGNIFIER_SIZE);
ctx.restore();
// Border circle
ctx.beginPath();
ctx.arc(MAGNIFIER_SIZE / 2, MAGNIFIER_SIZE / 2, MAGNIFIER_SIZE / 2 - 2, 0, Math.PI * 2);
ctx.strokeStyle = "#3cabe2";
ctx.lineWidth = 3;
ctx.stroke();
// Crosshair
ctx.beginPath();
ctx.moveTo(MAGNIFIER_SIZE / 2 - 8, MAGNIFIER_SIZE / 2);
ctx.lineTo(MAGNIFIER_SIZE / 2 + 8, MAGNIFIER_SIZE / 2);
ctx.moveTo(MAGNIFIER_SIZE / 2, MAGNIFIER_SIZE / 2 - 8);
ctx.lineTo(MAGNIFIER_SIZE / 2, MAGNIFIER_SIZE / 2 + 8);
ctx.strokeStyle = "rgba(255,255,255,0.8)";
ctx.lineWidth = 1;
ctx.stroke();
};
/**
* @returns A crop point to use during cropping and image rotation
*/
const CropPoint = ({ pointSize, pointArea, defaultPosition, onStop: externalOnStop, onDrag: externalOnDrag, bounds, cropPointStyles = {}, previewCanvasRef, }) => {
const { romaine: { cropPoints }, } = useRomaine();
const cropPointStyle = useMemo(() => {
cropPointStyles.width = pointSize;
cropPointStyles.height = pointSize;
return buildCropPointStyle(cropPointStyles);
}, [cropPointStyles, pointSize]);
const magnifierRef = useRef(null);
const [isDragging, setIsDragging] = useState(false);
const updateMagnifier = useCallback((pointX, pointY) => {
if (!magnifierRef.current || !previewCanvasRef?.current)
return;
drawMagnifier(magnifierRef.current, previewCanvasRef.current, pointX, pointY);
}, [previewCanvasRef]);
const onStart = useCallback((_, data) => {
setIsDragging(true);
updateMagnifier(data.x + pointSize / 2, data.y + pointSize / 2);
}, [pointSize, updateMagnifier]);
const onDrag = useCallback((_, position) => {
const px = position.x + pointSize / 2;
const py = position.y + pointSize / 2;
externalOnDrag({ ...position, x: px, y: py }, pointArea);
updateMagnifier(px, py);
}, [externalOnDrag, pointArea, pointSize, updateMagnifier]);
const onStop = useCallback((_, position) => {
setIsDragging(false);
externalOnStop({
...position,
x: position.x + pointSize / 2,
y: position.y + pointSize / 2,
}, pointArea, cropPoints);
}, [externalOnStop, cropPoints, pointArea, pointSize]);
const nodeRef = useRef(null);
// Position magnifier above or below with hysteresis to prevent flipping
const magnifierSide = useRef("above");
const pointPos = cropPoints[pointArea];
const threshold = MAGNIFIER_SIZE + MAGNIFIER_OFFSET;
const buffer = 40;
if (magnifierSide.current === "above" && pointPos.y < threshold) {
magnifierSide.current = "below";
}
else if (magnifierSide.current === "below" && pointPos.y > threshold + buffer) {
magnifierSide.current = "above";
}
const magX = pointPos.x - MAGNIFIER_SIZE / 2;
const magY = magnifierSide.current === "above"
? pointPos.y - MAGNIFIER_SIZE - MAGNIFIER_OFFSET
: pointPos.y + pointSize + MAGNIFIER_OFFSET;
return (jsxs(Fragment, { children: [jsx(Draggable, { nodeRef: nodeRef, bounds: bounds, defaultPosition: defaultPosition, position: {
x: pointPos.x - pointSize / 2,
y: pointPos.y - pointSize / 2,
}, onStart: onStart, onDrag: onDrag, onStop: onStop, children: jsx("div", { ref: nodeRef, style: cropPointStyle }) }), jsx("canvas", { ref: magnifierRef, width: MAGNIFIER_SIZE, height: MAGNIFIER_SIZE, style: {
position: "absolute",
left: magX,
top: magY,
width: MAGNIFIER_SIZE,
height: MAGNIFIER_SIZE,
borderRadius: "50%",
zIndex: 1002,
pointerEvents: "none",
boxShadow: "0 2px 10px rgba(0,0,0,0.3)",
display: isDragging ? "block" : "none",
} })] }));
};
const CropPoints = ({ previewDims, ...otherProps }) => {
return (jsxs(Fragment, { children: [jsx(CropPoint, { pointArea: "left-top", defaultPosition: { x: 0, y: 0 }, ...otherProps }), jsx(CropPoint, { pointArea: "right-top", defaultPosition: { x: previewDims.width, y: 0 }, ...otherProps }), jsx(CropPoint, { pointArea: "right-bottom", defaultPosition: { x: 0, y: previewDims.height }, ...otherProps }), jsx(CropPoint, { pointArea: "left-bottom", defaultPosition: {
x: previewDims.width,
y: previewDims.height,
}, ...otherProps })] }));
};
CropPoints.propTypes = {
previewDims: T.shape({
ratio: T.number,
width: T.number,
height: T.number,
}),
};
/**
* Create the lines for the cropper utility
*/
const CropPointsDelimiters = ({
// romaineRef,
crop, cropPoints, previewDims, lineWidth = 3, lineColor = "#3cabe2", pointSize, ...props }) => {
const canvas = useRef(null);
const clearCanvas = useCallback(() => {
if (canvas.current) {
const ctx = canvas.current.getContext("2d");
ctx && ctx.clearRect(0, 0, previewDims.width, previewDims.height);
}
}, [previewDims]);
/** This needs to do a better job. See Issue #2
* @link https://github.com/DanielBailey-web/romaine/issues/2
*
*/
const sortPoints = useCallback(() => {
const sortOrder = [
"left-top",
"right-top",
"right-bottom",
"left-bottom",
];
return sortOrder.reduce((acc, pointPos) => [...acc, cropPoints[pointPos]], []);
}, [cropPoints]);
const drawShape = useCallback(([point1, point2, point3, point4]) => {
const ctx = canvas.current && canvas.current.getContext("2d");
if (ctx) {
ctx.lineWidth = lineWidth;
ctx.strokeStyle = lineColor;
ctx.beginPath();
ctx.moveTo(point1.x + pointSize / 2, point1.y);
ctx.lineTo(point2.x - pointSize / 2, point2.y);
ctx.moveTo(point2.x, point2.y + pointSize / 2);
ctx.lineTo(point3.x, point3.y - pointSize / 2);
ctx.moveTo(point3.x - pointSize / 2, point3.y);
ctx.lineTo(point4.x + pointSize / 2, point4.y);
ctx.moveTo(point4.x, point4.y - pointSize / 2);
ctx.lineTo(point1.x, point1.y + pointSize / 2);
ctx.closePath();
ctx.stroke();
}
}, [lineColor, lineWidth, pointSize]);
useEffect(() => {
if (cropPoints && canvas.current) {
clearCanvas();
const sortedPoints = sortPoints();
drawShape(sortedPoints);
}
}, [cropPoints, drawShape, clearCanvas, sortPoints]);
/**
* Takes in a `CoordinateXY` and makes sure that it is inside the `ContourCoordinates`
*
* @returns boolean
*
* @todo Should use point slope formula for when using perspective cropper
*/
const xyInPoints = ({ x, y }) => {
if (x > cropPoints["left-top"].x &&
y > cropPoints["left-top"].y &&
x < cropPoints["right-top"].x &&
y > cropPoints["right-top"].y &&
x < cropPoints["right-bottom"].x &&
y < cropPoints["right-bottom"].y &&
x > cropPoints["left-bottom"].x &&
y < cropPoints["left-bottom"].y)
return true;
return false;
};
const getMousePosition = (event) => {
if (canvas?.current?.getBoundingClientRect) {
const x = event.clientX - canvas.current.getBoundingClientRect().left;
const y = event.clientY - canvas.current.getBoundingClientRect().top;
if (xyInPoints({ x, y }))
return "inside";
else
return "outside";
}
};
const [cursor, setCursor] = useState("default");
const handleMouseMove = (e) => {
const pos = getMousePosition(e);
if (pos === "inside" && cursor !== "crosshair")
setCursor("crosshair");
else if (pos === "outside" && cursor !== "default")
setCursor("default");
};
const handleClick = (e) => {
const pos = getMousePosition(e);
if (pos === "inside")
crop({
preview: true,
filterCvParams: {
grayScale: false,
th: false,
},
image: {
quality: 1,
type: "image/jpeg",
},
});
};
return (jsxs(Fragment, { children: [jsx("canvas", { id: `${props.saltId ? props.saltId + "-" : ""}crop-point-delimiters`, ref: canvas, style: {
position: "absolute",
top: 0,
right: 0,
left: 0,
bottom: 0,
zIndex: 5,
pointerEvents: "none",
}, width: previewDims.width, height: previewDims.height }), jsx("div", { style: {
position: "absolute",
top: 0,
right: 0,
left: 0,
bottom: 0,
zIndex: 4,
cursor,
}, onClick: handleClick, onMouseMove: handleMouseMove, onMouseLeave: () => setCursor("default") })] }));
};
CropPointsDelimiters.propTypes = {
previewDims: T.shape({
ratio: T.number,
width: T.number,
height: T.number,
}),
cropPoints: T.shape({
"left-top": T.shape({ x: T.number, y: T.number }).isRequired,
"right-top": T.shape({ x: T.number, y: T.number }).isRequired,
"right-bottom": T.shape({ x: T.number, y: T.number }).isRequired,
"left-bottom": T.shape({ x: T.number, y: T.number }).isRequired,
}),
lineColor: T.string,
lineWidth: T.number,
pointSize: T.number,
saltId: T.string,
};
const CroppingCanvas = ({ image, onDragStop, onChange, previewCanvasRef, setCropFunc, pointSize = 30, lineWidth, lineColor, previewDims, imageResizeRatio, showPreview, setPreviewPaneDimensions, saltId, canvasPtr, }) => {
const { loaded: cvLoaded, cv, romaine: { mode, cropPoints }, setMode, pushHistory, setCropPoints, } = useRomaine();
// const [_worker, setWorker] = useState<Worker | null>();
// useEffect(() => {
// if (window.Worker) {
// const worker = new Worker("./test.js");
// worker.onmessage = function (e) {
// console.log(e.data);
// console.log("Message received from worker");
// };
// worker.postMessage([20, 30]);
// setWorker(worker);
// }
// }, []);
const [loading, setLoading] = useState(false);
const cropCB = useCallback(async (opts = {}) => {
// need to figure out how to not need this and still render
setLoading(true);
// push can be moved to mode
pushHistory?.();
return new Promise((resolve) => {
if (!opts.cropPoints)
opts.cropPoints = cropPoints;
if (!opts.imageResizeRatio)
opts.imageResizeRatio = imageResizeRatio;
if (!opts.mode)
opts.mode = mode;
if (opts.cropPoints) {
if (opts.mode === "crop")
cropOpenCV(cv, canvasPtr.current, opts.cropPoints, opts.imageResizeRatio
// canvasRef.current,
);
else if (opts.mode === "perspective-crop")
warpPerspective(cv, canvasPtr.current, opts.cropPoints, opts.imageResizeRatio);
else
return setLoading(false);
// @todo implement this somewhere
// applyFilter(cv, canvasRef.current, opts.filterCvParams);
if (opts.preview) {
const dims = canvasPtr.current.size();
const irr = setPreviewPaneDimensions(dims);
showPreview(irr, canvasPtr.current, false);
setMode?.(null);
}
setLoading(false);
return resolve();
}
});
},
// disabling because eslint doesn't know that canvasRef is a ref
// eslint-disable-next-line react-hooks/exhaustive-deps
[
cv,
cropPoints,
mode,
pushHistory,
setLoading,
setMode,
setPreviewPaneDimensions,
imageResizeRatio,
]);
useEffect(() => {
setCropFunc(() => cropCB);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [cropCB]);
const detectContours = () => {
/*
* warning, this function will alter the canvasPtr
*
* Remember to copy pointer to new variable if you want to keep it
* Make sure to clean up variable also
*/
const dst = new cv.Mat();
canvasPtr.current.copyTo(dst);
const ksize = new cv.Size(5, 5);
// convert the image to grayscale, blur it, and find edges in the image
cv.cvtColor(dst, dst, cv.COLOR_RGBA2GRAY, 0);
cv.GaussianBlur(dst, dst, ksize, 0, 0, cv.BORDER_DEFAULT);
cv.Canny(dst, dst, 75, 200);
// find contours
cv.threshold(dst, dst, 120, 200, cv.THRESH_BINARY);
const contours = new cv.MatVector();
const hierarchy = new cv.Mat();
cv.findContours(dst, contours, hierarchy, cv.RETR_CCOMP, cv.CHAIN_APPROX_SIMPLE);
const rect = cv.boundingRect(dst);
dst.delete();
hierarchy.delete();
contours.delete();
// transform the rectangle into a set of points
Object.keys(rect).forEach((key) => {
rect[key] = rect[key] * imageResizeRatio;
});
const contourCoordinates = {
"left-top": { x: rect.x, y: rect.y },
"right-top": { x: rect.x + rect.width, y: rect.y },
"right-bottom": {
x: rect.x + rect.width,
y: rect.y + rect.height,
},
"left-bottom": { x: rect.x, y: rect.y + rect.height },
};
setCropPoints(contourCoordinates);
};
useEffect(() => {
if (onChange && cropPoints) {
onChange({ ...cropPoints, loading });
}
}, [cropPoints, loading]);
/**
* @todo
* 1) Need to make sure that this is valid to never run the other code
* if it is, then need to remove the code in the else block
*/
const bootstrap = async () => {
detectContours();
setLoading(false);
// else {
// // imread is SLOW
// const src = cv.imread(previewCanvasRef.current);
// const contourCoordinates = {
// "left-top": { x: 0, y: 0 },
// "right-top": { x: src.cols, y: 0 },
// "right-bottom": {
// x: src.cols,
// y: src.rows,
// },
// "left-bottom": { x: 0, y: src.rows },
// };
// setCropPoints(contourCoordinates);
// }
};
useEffect(() => {
if (image &&
previewCanvasRef.current &&
cvLoaded &&
(mode === "crop" || mode === "perspective-crop")) {
bootstrap();
}
else {
setLoading(true);
}
}, [image, cvLoaded, mode]);
const handleNormalCornerMove = useCallback((position, area, cPs) => {
const { x, y } = position;
if (cPs) {
switch (area) {
case "left-bottom":
return {
...cPs,
[area]: { x, y },
"left-top": { x, y: cPs["left-top"].y },
"right-bottom": { x: cPs["right-bottom"].x, y },
};
case "right-bottom":
return {
...cPs,
[area]: { x, y },
"right-top": { x, y: cPs["right-top"].y },
"left-bottom": { x: cPs["left-bottom"].x, y },
};
case "right-top":
return {
...cPs,
[area]: { x, y },
"left-top": { x: cPs["left-top"].x, y },
"right-bottom": { x, y: cPs["right-bottom"].y },
};
case "left-top":
return {
...cPs,
[area]: { x, y },
"left-bottom": { x, y: cPs["left-bottom"].y },
"right-top": { x: cPs["right-top"].x, y },
};
}
}
}, []);
const onCornerDrag = useCallback((position, area) => {
// if (magnifierCanvasRef.current && previewCanvasRef.current) {
// const { x, y } = position;
// const magnCtx = magnifierCanvasRef.current.getContext("2d");
// clearCanvasByRef(magnifierCanvasRef);
// TODO we should make those 5, 10 and 20 values proportionate
// to the point size
// magnCtx &&
// magnCtx.drawImage(
// previewCanvasRef.current,
// x - (pointSize - 10),
// y - (pointSize - 10),
// pointSize + 5,
// pointSize + 5,
// x + 10,
// y - 90,
// pointSize + 20,
// pointSize + 20
// );
setCropPoints((cPs) => {
if (cPs && mode === "perspective-crop")
return { ...cPs, [area]: position };
return handleNormalCornerMove(position, area, cPs);
});
// }
}, [mode, setCropPoints, handleNormalCornerMove]);
const onCornerStop = useCallback((position, area, cropPoints) => {
const { x, y } = position;
// clearCanvasByRef(magnifierCanvasRef);
setCropPoints((cPs) => {
if (cPs && mode === "perspective-crop")
return { ...cPs, [area]: { x, y } };
return handleNormalCornerMove(position, area, cPs);
});
onDragStop?.({ ...cropPoints, [area]: { x, y }, loading });
}, [mode, setCropPoints, handleNormalCornerMove]);
return (jsx(Fragment, { children: previewDims &&
(mode === "crop" || mode === "perspective-crop") &&
cropPoints && (jsxs(Fragment, { children: [jsx(CropPoints, { pointSize: pointSize, previewDims: previewDims, onDrag: onCornerDrag, onStop: onCornerStop, previewCanvasRef: previewCanvasRef, bounds: {
left: (previewCanvasRef?.current?.offsetLeft || 0) - pointSize / 2,
top: (previewCanvasRef?.current?.offsetTop || 0) - pointSize / 2,
right: (previewCanvasRef?.current?.offsetLeft || 0) -
pointSize / 2 +
(previewCanvasRef?.current?.offsetWidth || 0),
bottom: (previewCanvasRef?.current?.offsetTop || 0) -
pointSize / 2 +
(previewCanvasRef?.current?.offsetHeight || 0),
} }), jsx(CropPointsDelimiters
// romaineRef={romaineRef as React.RefObject<RomaineRef>}
, {
// romaineRef={romaineRef as React.RefObject<RomaineRef>}
crop: cropCB, previewDims: previewDims, cropPoints: cropPoints, lineWidth: lineWidth, lineColor: lineColor, pointSize: pointSize, saltId: saltId })] })) }));
};
const buildImgContainerStyle = (previewDims) => ({
width: previewDims.width,
height: previewDims.height,
});
const usePreview = ({ canvasRef, maxDims, originalImageDims, }) => {
const { maxHeight, maxWidth } = maxDims;
const { cv } = useRomaine();
const previewRef = useRef(null);
const [imageResizeRatio, setImageResizeRatio] = useState(1);
const [previewDims, setPreviewDims] = useState({
height: maxHeight,
width: maxWidth,
ratio: 1,
});
const createPreview = (resizeRatio = imageResizeRatio, source, cleanup = true) => {
if (!source && canvasRef.current)
source == cv.imread(canvasRef.current);
if (!source?.$$.ptr)
return;
if (cv && previewRef.current) {
const dst = new cv.Mat();
const dsize = new cv.Size(0, 0);
cv.resize(source, dst, dsize, resizeRatio, resizeRatio, cv.INTER_AREA);
cv.imshow(previewRef.current, dst);
if (cleanup)
source.delete();
dst.delete();
}
};
const setPreviewPaneDimensions = useCallback((dims = originalImageDims) => {
if (dims && previewRef?.current) {
const newPreviewDims = calcDims(dims.width, dims.height, maxWidth, maxHeight);
setPreviewDims(newPreviewDims);
previewRef.current.width = newPreviewDims.width;
previewRef.current.height = newPreviewDims.height;
const irr = newPreviewDims.width / dims.width;
setImageResizeRatio(irr);
return irr;
}
}, [maxHeight, maxWidth]);
return {
previewRef,
createPreview,
imageResizeRatio,
setPreviewPaneDimensions,
previewDims,
};
};
const useCanvas = ({ image, saltId }) => {
const { cv, romaine: { mode }, updateImageInformation, } = useRomaine();
const [loaded, setLoaded] = useState(false);
const [originalImageDims, setOriginalImageDims] = useState({
width: 0,
height: 0,
});
const canvasRef = useRef(document.createElement("canvas"));
const canvasPtr = useRef(undefined);
const getFile = useCallback(async (image) => await readFile(image), []);
const createCanvas = useCallback(() => new Promise(async (resolve, reject) => {
if (!image)
return reject("Image source is invalid");
const src = await getFile(image);
try {
const img = document.createElement("img");
img.style.display = "none";
img.id = `${saltId ? saltId + "-" : ""}working-image`;
img.onload = async () => {
// set edited image canvas and dimensions
canvasRef.current = document.createElement("canvas");
canvasRef.current.style.display = "none";
canvasRef.current.id = `${saltId ? saltId + "-" : ""}working-canvas`;
canvasRef.current.width = img.width;
canvasRef.current.height = img.height;
setOriginalImageDims({ height: img.height, width: img.width });
updateImageInformation?.({
height: img.height,
width: img.width,
id: img.id,
});
const ctx = canvasRef.current.getContext("2d");
if (ctx) {
ctx.fillStyle = "#fff0"; // transparent
ctx.fillRect(0, 0, img.width, img.height);
ctx.drawImage(img, 0, 0);
canvasPtr.current = cv.imread(canvasRef.current);
setLoaded(true);
return resolve();
}
return reject();
};
if (isCrossOriginURL(src))
img.crossOrigin = "anonymous";
img.src = src;
}
catch (err) {
reject("unknown error while creating canvas");
}
}), [image]);
const resetImage = useCallback(() => {
// set loaded to false before we destroy the pointer to avoid race conditions
setLoaded(false);
canvasPtr.current?.delete();
createCanvas();
}, [image]);
useEffect(() => {
if (image)
resetImage();
}, [image]);
useEffect(() => {
if (mode === "undo") {
resetImage();
}
}, [mode]);
return {
canvasRef,
createCanvas,
originalImageDims,
canvasPtr,
resetImage,
/** loaded: Is the image loaded onto the canvas and does the pointer exist */
loaded,
};
};
const flip = async (cv, canvas, src, orientation) => {
if (orientation === "horizontal") {
cv.flip(src, src, 0);
}
else if (orientation === "vertical") {
cv.flip(src, src, 1);
}
else {
throw new Error("Invalid orientation");
}
cv.imshow(canvas, src);
};
const rotate = async (cv, canvas, mode, setPreviewPaneDimensions, show