From eac523c83f8b67488ae122b6e31c1f2f88bf1c45 Mon Sep 17 00:00:00 2001 From: dwelle <5153846+dwelle@users.noreply.github.com> Date: Mon, 24 Jun 2024 22:59:31 +0200 Subject: [PATCH] wip --- package.json | 1 + .../actions/actionRemoveBackground.tsx | 113 ++++++++++++++++++ packages/excalidraw/actions/index.ts | 1 + packages/excalidraw/actions/types.ts | 3 +- packages/excalidraw/components/Actions.tsx | 5 + packages/excalidraw/data/blob.ts | 4 +- packages/excalidraw/types.ts | 1 + yarn.lock | 87 +++++++++++++- 8 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 packages/excalidraw/actions/actionRemoveBackground.tsx diff --git a/package.json b/package.json index 7f8b73de2..6b1b8520c 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ ], "dependencies": { "@excalidraw/random-username": "1.0.0", + "@imgly/background-removal": "1.5.3", "@sentry/browser": "6.2.5", "@sentry/integrations": "6.2.5", "firebase": "8.3.3", diff --git a/packages/excalidraw/actions/actionRemoveBackground.tsx b/packages/excalidraw/actions/actionRemoveBackground.tsx new file mode 100644 index 000000000..b0099bdfe --- /dev/null +++ b/packages/excalidraw/actions/actionRemoveBackground.tsx @@ -0,0 +1,113 @@ +import { generateIdFromFile, getDataURL } from "../data/blob"; +import { mutateElement } from "../element/mutateElement"; +import { isInitializedImageElement } from "../element/typeChecks"; +import type { InitializedExcalidrawImageElement } from "../element/types"; +import type { BinaryFileData } from "../types"; +import { register } from "./register"; + +export const actionRemoveBackground = register({ + name: "removeBackground", + label: "stats.fullTitle", + trackEvent: false, + viewMode: false, + async perform(elements, appState, _, app) { + const selectedElements = app.scene.getSelectedElements(appState); + + if ( + selectedElements.length > 0 && + selectedElements.every(isInitializedImageElement) + ) { + const filesToProcess = selectedElements.reduce( + ( + acc: Map< + BinaryFileData["id"], + { + file: BinaryFileData; + elements: InitializedExcalidrawImageElement[]; + } + >, + imageElement, + ) => { + const file = app.files[imageElement.fileId]; + + if (file) { + const fileWithRemovedBackground = Object.values(app.files).find( + (_file) => + _file.customData?.source === "backgroundRemoval" && + _file.customData.parentFileId === file.id, + ); + + if (fileWithRemovedBackground) { + mutateElement( + imageElement, + { fileId: fileWithRemovedBackground.id }, + false, + ); + } else if (acc.has(file.id)) { + acc.get(file.id)!.elements.push(imageElement); + } else { + acc.set(file.id, { file, elements: [imageElement] }); + } + } + return acc; + }, + new Map(), + ); + + if (filesToProcess.size) { + const backgroundRemoval = await await import( + "@imgly/background-removal" + ); + + console.time("removeBackground"); + + for (const [, { file, elements }] of filesToProcess) { + const res = await backgroundRemoval.removeBackground(file.dataURL, { + // debug: true, + progress: (...args) => { + console.log("progress", args); + }, + device: "gpu", + proxyToWorker: true, + }); + + const fileId = await generateIdFromFile(res); + const dataURL = await getDataURL(res); + + for (const imageElement of elements) { + mutateElement(imageElement, { fileId }, false); + } + + app.addFiles([ + { + ...file, + id: fileId, + dataURL, + customData: { + source: "backgroundRemoval", + version: 1, + parentFileId: file.id, + }, + }, + ]); + } + + console.timeEnd("removeBackground"); + } + + app.scene.triggerUpdate(); + } + return false as false; + }, + PanelComponent: ({ updateData }) => { + return ( + + ); + }, +}); diff --git a/packages/excalidraw/actions/index.ts b/packages/excalidraw/actions/index.ts index 092060425..13bdf92b2 100644 --- a/packages/excalidraw/actions/index.ts +++ b/packages/excalidraw/actions/index.ts @@ -86,3 +86,4 @@ export { actionUnbindText, actionBindText } from "./actionBoundText"; export { actionLink } from "./actionLink"; export { actionToggleElementLock } from "./actionElementLock"; export { actionToggleLinearEditor } from "./actionLinearEditor"; +export { actionRemoveBackground } from "./actionRemoveBackground"; diff --git a/packages/excalidraw/actions/types.ts b/packages/excalidraw/actions/types.ts index 6597ec0f0..e73c9cac2 100644 --- a/packages/excalidraw/actions/types.ts +++ b/packages/excalidraw/actions/types.ts @@ -136,7 +136,8 @@ export type ActionName = | "wrapTextInContainer" | "commandPalette" | "autoResize" - | "elementStats"; + | "elementStats" + | "removeBackground"; export type PanelComponentProps = { elements: readonly ExcalidrawElement[]; diff --git a/packages/excalidraw/components/Actions.tsx b/packages/excalidraw/components/Actions.tsx index c49b4a5f0..0420bc05f 100644 --- a/packages/excalidraw/components/Actions.tsx +++ b/packages/excalidraw/components/Actions.tsx @@ -25,6 +25,7 @@ import { hasStrokeColor } from "../scene/comparisons"; import { trackEvent } from "../analytics"; import { hasBoundTextElement, + isInitializedImageElement, isLinearElement, isTextElement, } from "../element/typeChecks"; @@ -125,6 +126,10 @@ export const SelectedShapeActions = ({ return (
+ {targetElements.length > 0 && + targetElements.every(isInitializedImageElement) && ( +
{renderAction("removeBackground")}
+ )}
{canChangeStrokeColor(appState, targetElements) && renderAction("changeStrokeColor")} diff --git a/packages/excalidraw/data/blob.ts b/packages/excalidraw/data/blob.ts index 1eb1e1bed..bd46c1adc 100644 --- a/packages/excalidraw/data/blob.ts +++ b/packages/excalidraw/data/blob.ts @@ -235,7 +235,9 @@ export const canvasToBlob = async ( /** generates SHA-1 digest from supplied file (if not supported, falls back to a 40-char base64 random id) */ -export const generateIdFromFile = async (file: File): Promise => { +export const generateIdFromFile = async ( + file: File | Blob, +): Promise => { try { const hashBuffer = await window.crypto.subtle.digest( "SHA-1", diff --git a/packages/excalidraw/types.ts b/packages/excalidraw/types.ts index cb428f695..fe0c09e93 100644 --- a/packages/excalidraw/types.ts +++ b/packages/excalidraw/types.ts @@ -108,6 +108,7 @@ export type BinaryFileData = { * Epoch timestamp in milliseconds. */ lastRetrieved?: number; + customData?: Record; }; export type BinaryFileMetadata = Omit; diff --git a/yarn.lock b/yarn.lock index b9124e646..46807c497 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2238,6 +2238,19 @@ resolved "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== +"@imgly/background-removal@1.5.3": + version "1.5.3" + resolved "https://registry.yarnpkg.com/@imgly/background-removal/-/background-removal-1.5.3.tgz#5fb39eb97e0f26fefd6a270b18f771c1c905593d" + integrity sha512-Q5DI5EtOvTvsWueimB0XUlkDObdcQsN2hTEsQUnJXym7x7oH8dn1qrOZ6UklJSIHK7hkiqKtaDONvvk0lKVWmA== + dependencies: + "@types/lodash-es" "^4.17.12" + "@types/ndarray" "~1.0.14" + "@types/node" "~20.3.0" + lodash-es "^4.17.21" + ndarray "~1.0.0" + onnxruntime-web "~1.18.0" + zod "^3.23.8" + "@istanbuljs/schema@^0.1.2": version "0.1.3" resolved "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz#e45e384e4b8ec16bce2fd903af78450f6bf7ec98" @@ -3193,6 +3206,13 @@ resolved "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== +"@types/lodash-es@^4.17.12": + version "4.17.12" + resolved "https://registry.yarnpkg.com/@types/lodash-es/-/lodash-es-4.17.12.tgz#65f6d1e5f80539aa7cfbfc962de5def0cf4f341b" + integrity sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ== + dependencies: + "@types/lodash" "*" + "@types/lodash.throttle@4.1.7": version "4.1.7" resolved "https://registry.npmjs.org/@types/lodash.throttle/-/lodash.throttle-4.1.7.tgz#4ef379eb4f778068022310ef166625f420b6ba58" @@ -3222,6 +3242,11 @@ resolved "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz#10964ba0dee6ac4cd462e2795b6bebd407303433" integrity sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g== +"@types/ndarray@~1.0.14": + version "1.0.14" + resolved "https://registry.yarnpkg.com/@types/ndarray/-/ndarray-1.0.14.tgz#96b28c09a3587a76de380243f87bb7a2d63b4b23" + integrity sha512-oANmFZMnFQvb219SSBIhI1Ih/r4CvHDOzkWyJS/XRqkMrGH5/kaPSA1hQhdIBzouaE+5KpE/f5ylI9cujmckQg== + "@types/node@*", "@types/node@>=13.7.0", "@types/node@^20": version "20.12.4" resolved "https://registry.npmjs.org/@types/node/-/node-20.12.4.tgz#af5921bd75ccdf3a3d8b3fa75bf3d3359268cd11" @@ -3229,6 +3254,11 @@ dependencies: undici-types "~5.26.4" +"@types/node@~20.3.0": + version "20.3.3" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.3.3.tgz#329842940042d2b280897150e023e604d11657d6" + integrity sha512-wheIYdr4NYML61AjC8MKj/2jrR/kDQri/CIpVoZwldwhnIrD/j9jIU5bJ8yBKuB2VhpFV7Ab6G2XkBjv9r9Zzw== + "@types/pako@1.0.3": version "1.0.3" resolved "https://registry.npmjs.org/@types/pako/-/pako-1.0.3.tgz#2e61c2b02020b5f44e2e5e946dfac74f4ec33c58" @@ -6406,6 +6436,11 @@ flat@^5.0.2: resolved "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== +flatbuffers@^1.12.0: + version "1.12.0" + resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-1.12.0.tgz#72e87d1726cb1b216e839ef02658aa87dcef68aa" + integrity sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ== + flatted@^3.2.7, flatted@^3.2.9: version "3.3.1" resolved "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz#21db470729a6734d4997002f439cb308987f567a" @@ -6659,6 +6694,11 @@ graphemer@^1.4.0: resolved "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== +guid-typescript@^1.0.9: + version "1.0.9" + resolved "https://registry.yarnpkg.com/guid-typescript/-/guid-typescript-1.0.9.tgz#e35f77003535b0297ea08548f5ace6adb1480ddc" + integrity sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ== + gzip-size@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz#065367fd50c239c0671cbcbad5be3e2eeb10e462" @@ -6967,6 +7007,11 @@ invariant@^2.2.2, invariant@^2.2.4: dependencies: loose-envify "^1.0.0" +iota-array@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/iota-array/-/iota-array-1.0.0.tgz#81ef57fe5d05814cd58c2483632a99c30a0e8087" + integrity sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA== + is-arguments@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz#15b3f88fda01f2a97fec84ca761a560f123efa9b" @@ -7017,6 +7062,11 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" +is-buffer@^1.0.2: + version "1.1.6" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" + integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== + is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.7: version "1.2.7" resolved "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055" @@ -7713,7 +7763,7 @@ long@^4.0.0: resolved "https://registry.npmjs.org/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== -long@^5.0.0: +long@^5.0.0, long@^5.2.3: version "5.2.3" resolved "https://registry.npmjs.org/long/-/long-5.2.3.tgz#a3ba97f3877cf1d778eccbcb048525ebb77499e1" integrity sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q== @@ -8213,6 +8263,14 @@ natural-compare@^1.4.0: resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== +ndarray@~1.0.0: + version "1.0.19" + resolved "https://registry.yarnpkg.com/ndarray/-/ndarray-1.0.19.tgz#6785b5f5dfa58b83e31ae5b2a058cfd1ab3f694e" + integrity sha512-B4JHA4vdyZU30ELBw3g7/p9bZupyew5a7tX1Y/gGeF2hafrPaQZhgrGQfsvgfYbgdFZjYwuEcnaobeM/WMW+HQ== + dependencies: + iota-array "^1.0.0" + is-buffer "^1.0.2" + neo-async@^2.6.2: version "2.6.2" resolved "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" @@ -8415,6 +8473,23 @@ onetime@^6.0.0: dependencies: mimic-fn "^4.0.0" +onnxruntime-common@1.18.0: + version "1.18.0" + resolved "https://registry.yarnpkg.com/onnxruntime-common/-/onnxruntime-common-1.18.0.tgz#b904dc6ff134e7f21a3eab702fac17538f59e116" + integrity sha512-lufrSzX6QdKrktAELG5x5VkBpapbCeS3dQwrXbN0eD9rHvU0yAWl7Ztju9FvgAKWvwd/teEKJNj3OwM6eTZh3Q== + +onnxruntime-web@~1.18.0: + version "1.18.0" + resolved "https://registry.yarnpkg.com/onnxruntime-web/-/onnxruntime-web-1.18.0.tgz#cd46268d9472f89697da0a3282f13129f0acbfa0" + integrity sha512-o1UKj4ABIj1gmG7ae0RKJ3/GT+3yoF0RRpfDfeoe0huzRW4FDRLfbkDETmdFAvnJEXuYDE0YT+hhkia0352StQ== + dependencies: + flatbuffers "^1.12.0" + guid-typescript "^1.0.9" + long "^5.2.3" + onnxruntime-common "1.18.0" + platform "^1.3.6" + protobufjs "^7.2.4" + open-color@1.9.1: version "1.9.1" resolved "https://registry.npmjs.org/open-color/-/open-color-1.9.1.tgz#a6e6328f60eff7aa60e3e8fcfa50f53ff3eece35" @@ -8627,6 +8702,11 @@ pkg-types@^1.0.3: mlly "^1.2.0" pathe "^1.1.0" +platform@^1.3.6: + version "1.3.6" + resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7" + integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg== + png-chunk-text@1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/png-chunk-text/-/png-chunk-text-1.0.0.tgz#1c6006d8e34ba471d38e1c9c54b3f53e1085e18f" @@ -11091,6 +11171,11 @@ yocto-queue@^1.0.0: resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz#7f816433fb2cbc511ec8bf7d263c3b58a1a3c251" integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g== +zod@^3.23.8: + version "3.23.8" + resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d" + integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g== + zustand@^4.3.2: version "4.5.2" resolved "https://registry.npmjs.org/zustand/-/zustand-4.5.2.tgz#fddbe7cac1e71d45413b3682cdb47b48034c3848"