Dylan/excalidraw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add text-to-drawing

Browse Source
This commit is contained in:
are 2023-11-28 18:35:03 +01:00
parent 0958241589
commit 530e92189f
No known key found for this signature in database
GPG Key ID: 8367A69658056EE3
12 changed files with 557 additions and 850 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
excalidraw-app/index.tsx
View File

@ -104,6 +104,7 @@ import { ShareableLinkDialog } from "../src/components/ShareableLinkDialog";
import { openConfirmModal } from "../src/components/OverwriteConfirm/OverwriteConfirmState";
import { OverwriteConfirmDialog } from "../src/components/OverwriteConfirm/OverwriteConfirm";
import Trans from "../src/components/Trans";
import { drawingIcon } from "../src/components/icons";
polyfill();
@ -776,12 +777,10 @@ const ExcalidrawWrapper = () => {
</OverwriteConfirmDialog>
<AppFooter />
<TTDDialog
onTextSubmit={async (input) => {
onTextSubmit={async (input, type) => {
try {
const response = await fetch(
`${
import.meta.env.VITE_APP_AI_BACKEND
}/v1/ai/text-to-diagram/generate`,
`${import.meta.env.VITE_APP_AI_BACKEND}/v1/ai/${type}/generate`,
{
method: "POST",
headers: {
@ -833,6 +832,9 @@ const ExcalidrawWrapper = () => {
}}
/>
<TTDDialogTrigger />
<TTDDialogTrigger tab="text-to-drawing" icon={drawingIcon}>
{t("labels.textToDrawing")}
</TTDDialogTrigger>
{isCollaborating && isOffline && (
<div className="collab-offline-warning">
{t("alerts.collabOfflineWarning")}

1
src/components/App.tsx
View File

@ -397,7 +397,6 @@ import { COLOR_PALETTE } from "../colors";
import { ElementCanvasButton } from "./MagicButton";
import { MagicIcon, copyIcon, fullscreenIcon } from "./icons";
import { EditorLocalStorage } from "../data/EditorLocalStorage";
import { TextToExcalidraw } from "./TextToExcalidraw/TextToExcalidraw";
const AppContext = React.createContext<AppClassProperties>(null!);
const AppPropsContext = React.createContext<AppProps>(null!);

2
src/components/TTDDialog/MermaidToExcalidraw.tsx
View File

@ -111,7 +111,7 @@ const MermaidToExcalidraw = ({
action: () => {
insertToEditor({
app,
data,
data: data.current,
text,
shouldSaveMermaidDataToStorage: true,
});

266
src/components/TTDDialog/TTDDialog.tsx
View File

@ -2,58 +2,20 @@ import { Dialog } from "../Dialog";
import { useApp } from "../App";
import MermaidToExcalidraw from "./MermaidToExcalidraw";
import TTDDialogTabs from "./TTDDialogTabs";
import { ChangeEventHandler, useEffect, useRef, useState } from "react";
import { useEffect, useState } from "react";
import { useUIAppState } from "../../context/ui-appState";
import { withInternalFallback } from "../hoc/withInternalFallback";
import { TTDDialogTabTriggers } from "./TTDDialogTabTriggers";
import { TTDDialogTabTrigger } from "./TTDDialogTabTrigger";
import { TTDDialogTab } from "./TTDDialogTab";
import { t } from "../../i18n";
import { TTDDialogInput } from "./TTDDialogInput";
import { TTDDialogOutput } from "./TTDDialogOutput";
import { TTDDialogPanel } from "./TTDDialogPanel";
import { TTDDialogPanels } from "./TTDDialogPanels";
import {
MermaidToExcalidrawLibProps,
convertMermaidToExcalidraw,
insertToEditor,
saveMermaidDataToStorage,
} from "./common";
import { NonDeletedExcalidrawElement } from "../../element/types";
import { BinaryFiles } from "../../types";
import { ArrowRightIcon } from "../icons";
import { CommonDialogProps, MermaidToExcalidrawLibProps } from "./common";
import "./TTDDialog.scss";
import { isFiniteNumber } from "../../utils";
import { atom, useAtom } from "jotai";
import { trackEvent } from "../../analytics";
import { TextToDiagram } from "./TextToDiagram";
import { TextToDrawing } from "./TextToDrawing";
const MIN_PROMPT_LENGTH = 3;
const MAX_PROMPT_LENGTH = 1000;
const rateLimitsAtom = atom<{
rateLimit: number;
rateLimitRemaining: number;
} | null>(null);
type OnTestSubmitRetValue = {
rateLimit?: number | null;
rateLimitRemaining?: number | null;
} & (
| { generatedResponse: string | undefined; error?: null | undefined }
| {
error: Error;
generatedResponse?: null | undefined;
}
);
export const TTDDialog = (
props:
| {
onTextSubmit(value: string): Promise<OnTestSubmitRetValue>;
}
| { __fallback: true },
) => {
export const TTDDialog = (props: CommonDialogProps | { __fallback: true }) => {
const appState = useUIAppState();
if (appState.openDialog?.name !== "ttd") {
@ -72,118 +34,10 @@ export const TTDDialogBase = withInternalFallback(
tab,
...rest
}: {
tab: "text-to-diagram" | "mermaid";
} & (
| {
onTextSubmit(value: string): Promise<OnTestSubmitRetValue>;
}
| { __fallback: true }
)) => {
tab: "text-to-diagram" | "mermaid" | "text-to-drawing";
} & (CommonDialogProps | { __fallback: true })) => {
const app = useApp();
const someRandomDivRef = useRef<HTMLDivElement>(null);
const [text, setText] = useState("");
const prompt = text.trim();
const handleTextChange: ChangeEventHandler<HTMLTextAreaElement> = (
event,
) => {
setText(event.target.value);
};
const [onTextSubmitInProgess, setOnTextSubmitInProgess] = useState(false);
const [rateLimits, setRateLimits] = useAtom(rateLimitsAtom);
const onGenerate = async () => {
if (
prompt.length > MAX_PROMPT_LENGTH ||
prompt.length < MIN_PROMPT_LENGTH ||
onTextSubmitInProgess ||
rateLimits?.rateLimitRemaining === 0 ||
// means this is not a text-to-diagram dialog (needed for TS only)
"__fallback" in rest
) {
if (prompt.length < MIN_PROMPT_LENGTH) {
setError(
new Error(
`Prompt is too short (min ${MIN_PROMPT_LENGTH} characters)`,
),
);
}
if (prompt.length > MAX_PROMPT_LENGTH) {
setError(
new Error(
`Prompt is too long (max ${MAX_PROMPT_LENGTH} characters)`,
),
);
}
return;
}
try {
setOnTextSubmitInProgess(true);
trackEvent("ai", "generate", "ttd");
const { generatedResponse, error, rateLimit, rateLimitRemaining } =
await rest.onTextSubmit(prompt);
if (isFiniteNumber(rateLimit) && isFiniteNumber(rateLimitRemaining)) {
setRateLimits({ rateLimit, rateLimitRemaining });
}
if (error) {
setError(error);
return;
}
if (!generatedResponse) {
setError(new Error("Generation failed"));
return;
}
try {
await convertMermaidToExcalidraw({
canvasRef: someRandomDivRef,
data,
mermaidToExcalidrawLib,
setError,
mermaidDefinition: generatedResponse,
});
trackEvent("ai", "mermaid parse success", "ttd");
saveMermaidDataToStorage(generatedResponse);
} catch (error: any) {
console.info(
`%cTTD mermaid render errror: ${error.message}`,
"color: red",
);
console.info(
`>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\nTTD mermaid definition render errror: ${error.message}`,
"color: yellow",
);
trackEvent("ai", "mermaid parse failed", "ttd");
setError(
new Error(
"Generated an invalid diagram :(. You may also try a different prompt.",
),
);
}
} catch (error: any) {
let message: string | undefined = error.message;
if (!message || message === "Failed to fetch") {
message = "Request failed";
}
setError(new Error(message));
} finally {
setOnTextSubmitInProgess(false);
}
};
const refOnGenerate = useRef(onGenerate);
refOnGenerate.current = onGenerate;
const [mermaidToExcalidrawLib, setMermaidToExcalidrawLib] =
useState<MermaidToExcalidrawLibProps>({
loaded: false,
@ -200,13 +54,6 @@ export const TTDDialogBase = withInternalFallback(
fn();
}, [mermaidToExcalidrawLib.api]);
const data = useRef<{
elements: readonly NonDeletedExcalidrawElement[];
files: BinaryFiles | null;
}>({ elements: [], files: null });
const [error, setError] = useState<Error | null>(null);
return (
<Dialog
className="ttd-dialog"
@ -243,6 +90,9 @@ export const TTDDialogBase = withInternalFallback(
</div>
</div>
</TTDDialogTabTrigger>
<TTDDialogTabTrigger tab="text-to-drawing">
{t("labels.textToDrawing")}
</TTDDialogTabTrigger>
<TTDDialogTabTrigger tab="mermaid">Mermaid</TTDDialogTabTrigger>
</TTDDialogTabTriggers>
)}
@ -254,93 +104,15 @@ export const TTDDialogBase = withInternalFallback(
</TTDDialogTab>
{!("__fallback" in rest) && (
<TTDDialogTab className="ttd-dialog-content" tab="text-to-diagram">
<div className="ttd-dialog-desc">
Currently we use Mermaid as a middle step, so you'll get best
results if you describe a diagram, workflow, flow chart, and
similar.
</div>
<TTDDialogPanels>
<TTDDialogPanel
label={t("labels.prompt")}
panelAction={{
action: onGenerate,
label: "Generate",
icon: ArrowRightIcon,
}}
onTextSubmitInProgess={onTextSubmitInProgess}
panelActionDisabled={
prompt.length > MAX_PROMPT_LENGTH ||
rateLimits?.rateLimitRemaining === 0
}
renderTopRight={() => {
if (!rateLimits) {
return null;
}
return (
<div
className="ttd-dialog-rate-limit"
style={{
fontSize: 12,
marginLeft: "auto",
color:
rateLimits.rateLimitRemaining === 0
? "var(--color-danger)"
: undefined,
}}
>
{rateLimits.rateLimitRemaining} requests left today
</div>
);
}}
renderBottomRight={() => {
const ratio = prompt.length / MAX_PROMPT_LENGTH;
if (ratio > 0.8) {
return (
<div
style={{
marginLeft: "auto",
fontSize: 12,
fontFamily: "monospace",
color:
ratio > 1 ? "var(--color-danger)" : undefined,
}}
>
Length: {prompt.length}/{MAX_PROMPT_LENGTH}
</div>
);
}
return null;
}}
>
<TTDDialogInput
onChange={handleTextChange}
input={text}
placeholder={"Describe what you want to see..."}
onKeyboardSubmit={() => {
refOnGenerate.current();
}}
/>
</TTDDialogPanel>
<TTDDialogPanel
label="Preview"
panelAction={{
action: () => {
console.info("Panel action clicked");
insertToEditor({ app, data });
},
label: "Insert",
icon: ArrowRightIcon,
}}
>
<TTDDialogOutput
canvasRef={someRandomDivRef}
error={error}
loaded={mermaidToExcalidrawLib.loaded}
/>
</TTDDialogPanel>
</TTDDialogPanels>
<TextToDiagram
onTextSubmit={rest.onTextSubmit}
mermaidToExcalidrawLib={mermaidToExcalidrawLib}
/>
</TTDDialogTab>
)}
{!("__fallback" in rest) && (
<TTDDialogTab className="ttd-dialog-content" tab="text-to-drawing">
<TextToDrawing onTextSubmit={rest.onTextSubmit} />
</TTDDialogTab>
)}
</TTDDialogTabs>

7
src/components/TTDDialog/TTDDialogTabs.tsx
View File

@ -7,8 +7,11 @@ const TTDDialogTabs = (
props: {
children: ReactNode;
} & (
| { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" }
| { dialog: "settings"; tab: "text-to-diagram" | "diagram-to-code" }
| { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" | "text-to-drawing" }
| {
dialog: "settings";
tab: "text-to-diagram" | "diagram-to-code";
}
),
) => {
const setAppState = useExcalidrawSetAppState();

6
src/components/TTDDialog/TTDDialogTrigger.tsx
View File

@ -9,9 +9,11 @@ import { trackEvent } from "../../analytics";
export const TTDDialogTrigger = ({
children,
icon,
tab,
}: {
children?: ReactNode;
icon?: JSX.Element;
tab?: string;
}) => {
const { TTDDialogTriggerTunnel } = useTunnels();
const setAppState = useExcalidrawSetAppState();
@ -21,7 +23,9 @@ export const TTDDialogTrigger = ({
<DropdownMenu.Item
onSelect={() => {
trackEvent("ai", "dialog open", "ttd");
setAppState({ openDialog: { name: "ttd", tab: "text-to-diagram" } });
setAppState({
openDialog: { name: "ttd", tab: tab ?? "text-to-diagram" },
});
}}
icon={icon ?? brainIcon}
>

228
src/components/TTDDialog/TextToDiagram.tsx Normal file
View File

@ -0,0 +1,228 @@
import { useAtom } from "jotai";
import { useRef, useState, ChangeEventHandler } from "react";
import { trackEvent } from "../../analytics";
import { t } from "../../i18n";
import { isFiniteNumber } from "../../utils";
import { ArrowRightIcon } from "../icons";
import { TTDDialogInput } from "./TTDDialogInput";
import { TTDDialogOutput } from "./TTDDialogOutput";
import { TTDDialogPanel } from "./TTDDialogPanel";
import { TTDDialogPanels } from "./TTDDialogPanels";
import {
CommonDialogProps,
MAX_PROMPT_LENGTH,
MIN_PROMPT_LENGTH,
MermaidToExcalidrawLibProps,
convertMermaidToExcalidraw,
insertToEditor,
rateLimitsAtom,
saveMermaidDataToStorage,
} from "./common";
import { useApp } from "../App";
import { NonDeletedExcalidrawElement } from "../../element/types";
import { BinaryFiles } from "../../types";
export type TextToDiagramProps = CommonDialogProps & {
mermaidToExcalidrawLib: MermaidToExcalidrawLibProps;
};
export const TextToDiagram = ({
onTextSubmit,
mermaidToExcalidrawLib,
}: TextToDiagramProps) => {
const app = useApp();
const someRandomDivRef = useRef<HTMLDivElement>(null);
const [text, setText] = useState("");
const prompt = text.trim();
const handleTextChange: ChangeEventHandler<HTMLTextAreaElement> = (event) => {
setText(event.target.value);
};
const [onTextSubmitInProgess, setOnTextSubmitInProgess] = useState(false);
const [rateLimits, setRateLimits] = useAtom(rateLimitsAtom);
const data = useRef<{
elements: readonly NonDeletedExcalidrawElement[];
files: BinaryFiles | null;
}>({ elements: [], files: null });
const [error, setError] = useState<Error | null>(null);
const onGenerate = async () => {
if (
prompt.length > MAX_PROMPT_LENGTH ||
prompt.length < MIN_PROMPT_LENGTH ||
onTextSubmitInProgess ||
rateLimits?.rateLimitRemaining === 0
) {
if (prompt.length < MIN_PROMPT_LENGTH) {
setError(
new Error(
`Prompt is too short (min ${MIN_PROMPT_LENGTH} characters)`,
),
);
}
if (prompt.length > MAX_PROMPT_LENGTH) {
setError(
new Error(`Prompt is too long (max ${MAX_PROMPT_LENGTH} characters)`),
);
}
return;
}
try {
setOnTextSubmitInProgess(true);
trackEvent("ai", "generate", "ttd");
const { generatedResponse, error, rateLimit, rateLimitRemaining } =
await onTextSubmit(prompt, "text-to-diagram");
if (isFiniteNumber(rateLimit) && isFiniteNumber(rateLimitRemaining)) {
setRateLimits({ rateLimit, rateLimitRemaining });
}
if (error) {
setError(error);
return;
}
if (!generatedResponse) {
setError(new Error("Generation failed"));
return;
}
try {
await convertMermaidToExcalidraw({
canvasRef: someRandomDivRef,
data,
mermaidToExcalidrawLib,
setError,
mermaidDefinition: generatedResponse,
});
trackEvent("ai", "mermaid parse success", "ttd");
saveMermaidDataToStorage(generatedResponse);
} catch (error: any) {
console.info(
`%cTTD mermaid render errror: ${error.message}`,
"color: red",
);
console.info(
`>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\nTTD mermaid definition render errror: ${error.message}`,
"color: yellow",
);
trackEvent("ai", "mermaid parse failed", "ttd");
setError(
new Error(
"Generated an invalid diagram :(. You may also try a different prompt.",
),
);
}
} catch (error: any) {
let message: string | undefined = error.message;
if (!message || message === "Failed to fetch") {
message = "Request failed";
}
setError(new Error(message));
} finally {
setOnTextSubmitInProgess(false);
}
};
const refOnGenerate = useRef(onGenerate);
refOnGenerate.current = onGenerate;
return (
<>
<div className="ttd-dialog-desc">
Currently we use Mermaid as a middle step, so you'll get best results if
you describe a diagram, workflow, flow chart, and similar.
</div>
<TTDDialogPanels>
<TTDDialogPanel
label={t("labels.prompt")}
panelAction={{
action: onGenerate,
label: "Generate",
icon: ArrowRightIcon,
}}
onTextSubmitInProgess={onTextSubmitInProgess}
panelActionDisabled={
prompt.length > MAX_PROMPT_LENGTH ||
rateLimits?.rateLimitRemaining === 0
}
renderTopRight={() => {
if (!rateLimits) {
return null;
}
return (
<div
className="ttd-dialog-rate-limit"
style={{
fontSize: 12,
marginLeft: "auto",
color:
rateLimits.rateLimitRemaining === 0
? "var(--color-danger)"
: undefined,
}}
>
{rateLimits.rateLimitRemaining} requests left today
</div>
);
}}
renderBottomRight={() => {
const ratio = prompt.length / MAX_PROMPT_LENGTH;
if (ratio > 0.8) {
return (
<div
style={{
marginLeft: "auto",
fontSize: 12,
fontFamily: "monospace",
color: ratio > 1 ? "var(--color-danger)" : undefined,
}}
>
Length: {prompt.length}/{MAX_PROMPT_LENGTH}
</div>
);
}
return null;
}}
>
<TTDDialogInput
onChange={handleTextChange}
input={text}
placeholder={"Describe what you want to see..."}
onKeyboardSubmit={() => {
refOnGenerate.current();
}}
/>
</TTDDialogPanel>
<TTDDialogPanel
label="Preview"
panelAction={{
action: () => {
console.info("Panel action clicked");
insertToEditor({ app, data: data.current });
},
label: "Insert",
icon: ArrowRightIcon,
}}
>
<TTDDialogOutput
canvasRef={someRandomDivRef}
error={error}
loaded={mermaidToExcalidrawLib.loaded}
/>
</TTDDialogPanel>
</TTDDialogPanels>
</>
);
};

248
src/components/TTDDialog/TextToDrawing.tsx Normal file
View File

@ -0,0 +1,248 @@
import { useAtom } from "jotai";
import { useRef, useState, ChangeEventHandler } from "react";
import { trackEvent } from "../../analytics";
import { NonDeletedExcalidrawElement } from "../../element/types";
import { t } from "../../i18n";
import { isFiniteNumber } from "../../utils";
import { useApp } from "../App";
import { ArrowRightIcon } from "../icons";
import { TTDDialogInput } from "./TTDDialogInput";
import { TTDDialogOutput } from "./TTDDialogOutput";
import { TTDDialogPanel } from "./TTDDialogPanel";
import { TTDDialogPanels } from "./TTDDialogPanels";
import {
CommonDialogProps,
MAX_PROMPT_LENGTH,
MIN_PROMPT_LENGTH,
insertToEditor,
rateLimitsAtom,
resetPreview,
} from "./common";
import {
convertToExcalidrawElements,
exportToCanvas,
} from "../../packages/excalidraw/index";
import { DEFAULT_EXPORT_PADDING } from "../../constants";
import { canvasToBlob } from "../../data/blob";
export type TextToDrawingProps = CommonDialogProps;
export const TextToDrawing = ({ onTextSubmit }: TextToDrawingProps) => {
const app = useApp();
const containerRef = useRef<HTMLDivElement>(null);
const [text, setText] = useState("");
const prompt = text.trim();
const handleTextChange: ChangeEventHandler<HTMLTextAreaElement> = (event) => {
setText(event.target.value);
};
const [onTextSubmitInProgess, setOnTextSubmitInProgess] = useState(false);
const [rateLimits, setRateLimits] = useAtom(rateLimitsAtom);
const [data, setData] = useState<
readonly NonDeletedExcalidrawElement[] | null
>(null);
const [error, setError] = useState<Error | null>(null);
const onGenerate = async () => {
if (
prompt.length > MAX_PROMPT_LENGTH ||
prompt.length < MIN_PROMPT_LENGTH ||
onTextSubmitInProgess ||
rateLimits?.rateLimitRemaining === 0
) {
if (prompt.length < MIN_PROMPT_LENGTH) {
setError(
new Error(
`Prompt is too short (min ${MIN_PROMPT_LENGTH} characters)`,
),
);
}
if (prompt.length > MAX_PROMPT_LENGTH) {
setError(
new Error(`Prompt is too long (max ${MAX_PROMPT_LENGTH} characters)`),
);
}
return;
}
try {
setOnTextSubmitInProgess(true);
trackEvent("ai", "generate", "text-to-drawing");
const { generatedResponse, error, rateLimit, rateLimitRemaining } =
await onTextSubmit(prompt, "text-to-drawing");
if (isFiniteNumber(rateLimit) && isFiniteNumber(rateLimitRemaining)) {
setRateLimits({ rateLimit, rateLimitRemaining });
}
if (error) {
setError(error);
return;
}
if (!generatedResponse) {
setError(new Error("Generation failed"));
return;
}
const canvasNode = containerRef.current;
const parent = canvasNode?.parentElement;
if (!canvasNode || !parent) {
return;
}
if (!text) {
resetPreview({ canvasRef: containerRef, setError });
return;
}
if (!Array.isArray(generatedResponse)) {
setError(new Error("Generation failed to return an array!"));
return;
}
try {
const elements = convertToExcalidrawElements(generatedResponse, {
regenerateIds: true,
});
setData(elements);
const canvas = await exportToCanvas({
elements,
files: null,
exportPadding: DEFAULT_EXPORT_PADDING,
maxWidthOrHeight:
Math.max(parent.offsetWidth, parent.offsetHeight) *
window.devicePixelRatio,
});
// if converting to blob fails, there's some problem that will
// likely prevent preview and export (e.g. canvas too big)
await canvasToBlob(canvas);
parent.style.background = "var(--default-bg-color)";
canvasNode.replaceChildren(canvas);
} catch (err: any) {
console.error(err);
parent.style.background = "var(--default-bg-color)";
if (text) {
setError(err);
}
throw err;
}
} catch (error: any) {
let message: string | undefined = error.message;
if (!message || message === "Failed to fetch") {
message = "Request failed";
}
setError(new Error(message));
} finally {
setOnTextSubmitInProgess(false);
}
};
const refOnGenerate = useRef(onGenerate);
refOnGenerate.current = onGenerate;
return (
<>
<div className="ttd-dialog-desc">This is text to drawing.</div>
<TTDDialogPanels>
<TTDDialogPanel
label={t("labels.prompt")}
panelAction={{
action: onGenerate,
label: "Generate",
icon: ArrowRightIcon,
}}
onTextSubmitInProgess={onTextSubmitInProgess}
panelActionDisabled={
prompt.length > MAX_PROMPT_LENGTH ||
rateLimits?.rateLimitRemaining === 0
}
renderTopRight={() => {
if (!rateLimits) {
return null;
}
return (
<div
className="ttd-dialog-rate-limit"
style={{
fontSize: 12,
marginLeft: "auto",
color:
rateLimits.rateLimitRemaining === 0
? "var(--color-danger)"
: undefined,
}}
>
{rateLimits.rateLimitRemaining} requests left today
</div>
);
}}
renderBottomRight={() => {
const ratio = prompt.length / MAX_PROMPT_LENGTH;
if (ratio > 0.8) {
return (
<div
style={{
marginLeft: "auto",
fontSize: 12,
fontFamily: "monospace",
color: ratio > 1 ? "var(--color-danger)" : undefined,
}}
>
Length: {prompt.length}/{MAX_PROMPT_LENGTH}
</div>
);
}
return null;
}}
>
<TTDDialogInput
onChange={handleTextChange}
input={text}
placeholder={"Describe what you want to see..."}
onKeyboardSubmit={() => {
refOnGenerate.current();
}}
/>
</TTDDialogPanel>
<TTDDialogPanel
label="Preview"
panelAction={{
action: () => {
if (data) {
insertToEditor({
app,
data: {
elements: data,
files: null,
},
});
}
},
label: "Insert",
icon: ArrowRightIcon,
}}
>
<TTDDialogOutput
canvasRef={containerRef}
error={error}
loaded={true}
/>
</TTDDialogPanel>
</TTDDialogPanels>
</>
);
};

37
src/components/TTDDialog/common.ts
View File

@ -8,8 +8,9 @@ import {
import { NonDeletedExcalidrawElement } from "../../element/types";
import { AppClassProperties, BinaryFiles } from "../../types";
import { canvasToBlob } from "../../data/blob";
import { atom } from "jotai";
const resetPreview = ({
export const resetPreview = ({
canvasRef,
setError,
}: {
@ -30,6 +31,26 @@ const resetPreview = ({
canvasNode.replaceChildren();
};
export type OnTestSubmitRetValue = {
rateLimit?: number | null;
rateLimitRemaining?: number | null;
} & (
| {
generatedResponse: any | string | undefined;
error?: null | undefined;
}
| {
error: Error;
generatedResponse?: null | undefined;
}
);
export interface CommonDialogProps {
onTextSubmit(
value: string,
type: "text-to-diagram" | "text-to-drawing",
): Promise<OnTestSubmitRetValue>;
}
export interface MermaidToExcalidrawLibProps {
loaded: boolean;
api: Promise<{
@ -137,14 +158,14 @@ export const insertToEditor = ({
shouldSaveMermaidDataToStorage,
}: {
app: AppClassProperties;
data: React.MutableRefObject<{
data: {
elements: readonly NonDeletedExcalidrawElement[];
files: BinaryFiles | null;
}>;
};
text?: string;
shouldSaveMermaidDataToStorage?: boolean;
}) => {
const { elements: newElements, files } = data.current;
const { elements: newElements, files } = data;
if (!newElements.length) {
return;
@ -162,3 +183,11 @@ export const insertToEditor = ({
saveMermaidDataToStorage(text);
}
};
export const MIN_PROMPT_LENGTH = 3;
export const MAX_PROMPT_LENGTH = 1000;
export const rateLimitsAtom = atom<{
rateLimit: number;
rateLimitRemaining: number;
} | null>(null);

589
src/components/TextToExcalidraw/TextToExcalidraw.tsx
View File

@ -1,589 +0,0 @@
import { useEffect, useRef, useState } from "react";
import { t } from "../../i18n";
import { useApp } from "../App";
import { Dialog } from "../Dialog";
import { TextField } from "../TextField";
import Trans from "../Trans";
import {
CloseIcon,
RedoIcon,
ZoomInIcon,
ZoomOutIcon,
playerPlayIcon,
playerStopFilledIcon,
} from "../icons";
import { NonDeletedExcalidrawElement } from "../../element/types";
import { convertToExcalidrawElements } from "../../data/transform";
import { exportToCanvas } from "../../packages/utils";
import { DEFAULT_EXPORT_PADDING } from "../../constants";
import { canvasToBlob } from "../../data/blob";
const testResponse = `{
"error": false,
"data": [
{
"type": "ellipse",
"x": 200,
"y": 200,
"width": 100,
"height": 100,
"strokeColor": "transparent",
"backgroundColor": "yellow",
"strokeWidth": 2
},
{
"type": "line",
"x": 300,
"y": 250,
"points": [
[
0,
0
],
[
70,
0
]
],
"width": -70,
"height": 0,
"strokeColor": "yellow",
"backgroundColor": "transparent",
"strokeWidth": 5
},
{
"type": "line",
"x": 293.30127018922195,
"y": 275,
"points": [
[
0,
0
],
[
60.62177826491069,
35
]
],
"width": -60.62177826491069,
"height": -35,
"strokeColor": "yellow",
"backgroundColor": "transparent",
"strokeWidth": 5
},
{
"type": "line",
"x": 275,
"y": 293.30127018922195,
"points": [
[
0,
0
],
[
35,
60.62177826491069
]
],
"width": -35,
"height": -60.62177826491069,
"strokeColor": "yellow",
"backgroundColor": "transparent",
"strokeWidth": 5
},
{
"type": "line",
"x": 250,
"y": 300,
"points": [
[
0,
0
],
[
0,
70
]
],
"width": 0,
"height": -70,
"strokeColor": "yellow",
"backgroundColor": "transparent",
"strokeWidth": 5
},
{
"type": "line",
"x": 225,
"y": 293.30127018922195,
"points": [
[
0,
0
],
[
-34.99999999999997,
60.62177826491069
]
],
"width": -34.99999999999997,
"height": -60.62177826491069,
"strokeColor": "yellow",
"backgroundColor": "transparent",
"strokeWidth": 5
},
{
"type": "line",
"x": 206.69872981077805,
"y": 275,
"points": [
[
0,
0
],
[
-60.62177826491069,
35
]
],
"width": -60.62177826491069,
"height": -35,
"strokeColor": "yellow",
"backgroundColor": "transparent",
"strokeWidth": 5
},
{
"type": "line",
"x": 200,
"y": 250,
"points": [
[
0,
0
],
[
-70,
2.842170943040401e-14
]
],
"width": -70,
"height": -2.842170943040401e-14,
"strokeColor": "yellow",
"backgroundColor": "transparent",
"strokeWidth": 5
},
{
"type": "line",
"x": 206.69872981077805,
"y": 225,
"points": [
[
0,
0
],
[
-60.62177826491069,
-34.99999999999997
]
],
"width": -60.62177826491069,
"height": -34.99999999999997,
"strokeColor": "yellow",
"backgroundColor": "transparent",
"strokeWidth": 5
},
{
"type": "line",
"x": 224.99999999999997,
"y": 206.69872981077808,
"points": [
[
0,
0
],
[
-35.00000000000003,
-60.62177826491069
]
],
"width": -35.00000000000003,
"height": -60.62177826491069,
"strokeColor": "yellow",
"backgroundColor": "transparent",
"strokeWidth": 5
},
{
"type": "line",
"x": 250,
"y": 200,
"points": [
[
0,
0
],
[
-2.842170943040401e-14,
-70
]
],
"width": -2.842170943040401e-14,
"height": -70,
"strokeColor": "yellow",
"backgroundColor": "transparent",
"strokeWidth": 5
},
{
"type": "line",
"x": 275,
"y": 206.69872981077808,
"points": [
[
0,
0
],
[
35,
-60.621778264910716
]
],
"width": -35,
"height": -60.621778264910716,
"strokeColor": "yellow",
"backgroundColor": "transparent",
"strokeWidth": 5
},
{
"type": "line",
"x": 293.3012701892219,
"y": 224.99999999999997,
"points": [
[
0,
0
],
[
60.621778264910745,
-35.00000000000003
]
],
"width": -60.621778264910745,
"height": -35.00000000000003,
"strokeColor": "yellow",
"backgroundColor": "transparent",
"strokeWidth": 5
}
]
}`;
async function fetchData(
prompt: string,
): Promise<readonly NonDeletedExcalidrawElement[]> {
const response = await fetch(
`http://localhost:3015/v1/ai/text-to-excalidraw/generate`,
{
method: "POST",
headers: {
Accept: "application/json",
"Content-Type": "application/json",
},
body: JSON.stringify({ prompt }),
},
);
const result = await response.json();
if (result.error) {
alert("Oops!");
return [];
}
return convertToExcalidrawElements(result.data);
}
export const TextToExcalidraw = () => {
const app = useApp();
const [prompt, setPrompt] = useState("");
const [isPanelOpen, setPanelOpen] = useState(false);
const [isLoading, setLoading] = useState(false);
const [data, setData] = useState<
readonly NonDeletedExcalidrawElement[] | null
>(null);
const [previewCanvas, setPreviewCanvas] = useState<HTMLCanvasElement | null>(
null,
);
const containerRef = useRef<HTMLDivElement>(null);
const onClose = () => {
app.setOpenDialog(null);
};
const onSubmit = async () => {
setPanelOpen(true);
setLoading(true);
const elements = await fetchData(prompt);
setData(elements);
const canvas = await exportToCanvas({
elements,
files: {},
exportPadding: DEFAULT_EXPORT_PADDING,
});
await canvasToBlob(canvas);
setPreviewCanvas(canvas);
setLoading(false);
};
const onInsert = async () => {
if (data) {
app.addElementsFromPasteOrLibrary({
elements: data,
files: {},
position: "center",
fitToContent: true,
});
onClose();
}
};
useEffect(() => {
if (containerRef.current && previewCanvas) {
containerRef.current.replaceChildren(previewCanvas);
}
}, [previewCanvas]);
// exportToCanvas([], {}, {}, {});
// exportToSvg([], {exportBackground}, {}, {})
return (
<div
style={{
position: "absolute",
top: "6.5rem",
pointerEvents: "auto",
width: "100%",
display: "flex",
flexDirection: "column",
gap: "0.75rem",
}}
>
<div
className="Island"
style={{
width: "100%",
display: "flex",
flexDirection: "row",
boxSizing: "border-box",
gap: "0.75rem",
alignItems: "center",
height: 48,
padding: "0.5rem",
}}
>
<input
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
type="text"
style={{
flexGrow: 1,
height: "100%",
boxSizing: "border-box",
border: 0,
outline: "none",
}}
placeholder="How can I help you today?"
/>
<button
style={{
cursor: "pointer",
height: "100%",
border: "none",
background: "white",
aspectRatio: "1/1",
padding: 0,
display: "flex",
justifyContent: "center",
alignItems: "center",
}}
>
<div
style={{ width: "1.25rem", height: "1.25rem", color: "#1B1B1F" }}
>
{CloseIcon}
</div>
</button>
<div
style={{ background: "#D6D6D6", width: 1, height: "1.5rem" }}
></div>
<button
style={{
cursor: "pointer",
height: "100%",
border: "none",
aspectRatio: "1/1",
padding: 0,
display: "flex",
justifyContent: "center",
alignItems: "center",
backgroundColor: "#6965DB",
borderRadius: "0.5rem",
}}
onClick={onSubmit}
>
<div style={{ width: "1.25rem", height: "1.25rem", color: "white" }}>
{isLoading ? playerStopFilledIcon : playerPlayIcon}
</div>
</button>
</div>
{isPanelOpen && (
<div
className="Island"
style={{
width: "100%",
display: "flex",
flexDirection: "row",
height: 400,
boxSizing: "border-box",
}}
>
{isLoading ? (
"loading"
) : (
<div
style={{
width: "100%",
height: "100%",
display: "flex",
flexDirection: "column",
}}
>
<div
ref={containerRef}
style={{
display: "flex",
alignItems: "center",
justifyContent: "center",
width: "100%",
height: "100%",
}}
/>
<div
style={{
borderTop: "1px solid #F0EFFF",
padding: "0.75rem",
display: "flex",
flexDirection: "row",
gap: "0.75rem",
}}
>
<button
style={{
cursor: "pointer",
width: 32,
height: "100%",
border: "none",
aspectRatio: "1/1",
padding: 0,
display: "flex",
justifyContent: "center",
alignItems: "center",
backgroundColor: "#F5F5F9",
borderRadius: "0.5rem",
}}
>
<div
style={{
width: "1.25rem",
height: "1.25rem",
color: "#1B1B1F",
}}
>
{RedoIcon}
</div>
</button>
<div style={{ width: 32, height: "100%", display: "flex" }}>
<button
style={{
cursor: "pointer",
width: 32,
height: "100%",
border: "none",
aspectRatio: "1/1",
padding: 0,
display: "flex",
justifyContent: "center",
alignItems: "center",
backgroundColor: "#F5F5F9",
borderRadius: "0.5rem 0 0 0.5rem",
}}
>
<div
style={{
width: "1.25rem",
height: "1.25rem",
color: "#1B1B1F",
}}
>
{ZoomOutIcon}
</div>
</button>
<button
style={{
cursor: "pointer",
width: 32,
height: "100%",
border: "none",
aspectRatio: "1/1",
padding: 0,
display: "flex",
justifyContent: "center",
alignItems: "center",
backgroundColor: "#F5F5F9",
borderRadius: "0 0.5rem 0.5rem 0",
}}
>
<div
style={{
width: "1.25rem",
height: "1.25rem",
color: "#1B1B1F",
}}
>
{ZoomInIcon}
</div>
</button>
</div>
<div style={{ flexGrow: 1 }}></div>
<button
style={{
cursor: "pointer",
height: "100%",
border: "none",
padding: "0.5rem 1rem",
display: "flex",
justifyContent: "center",
alignItems: "center",
backgroundColor: "#6965DB",
borderRadius: "0.5rem",
color: "white",
}}
onClick={onInsert}
>
Insert into scene &gt;
</button>
</div>
</div>
)}
</div>
)}
</div>
);
};

10
src/components/icons.tsx
View File

@ -1755,3 +1755,13 @@ export const brainIcon = createIcon(
</g>,
tablerIconProps,
);
export const drawingIcon = createIcon(
<g stroke="currentColor" fill="none">
<path stroke="none" d="M0 0h24v24H0z" fill="none" />
<path d="M20 17v-12c0 -1.121 -.879 -2 -2 -2s-2 .879 -2 2v12l2 2l2 -2z" />
<path d="M16 7h4" />
<path d="M18 19h-13a2 2 0 1 1 0 -4h4a2 2 0 1 0 0 -4h-3" />
</g>,
tablerIconProps,
);

3
src/locales/en.json
View File

@ -134,7 +134,8 @@
"removeAllElementsFromFrame": "Remove all elements from frame",
"eyeDropper": "Pick color from canvas",
"textToDiagram": "Text to diagram",
"prompt": "Prompt"
"prompt": "Prompt",
"textToDrawing": "Text to drawing"
},
"library": {
"noItems": "No items added yet...",