feat: add GET /api/v2/scenes endpoint

- load.controller.ts exposing listScenes()
- adjusted AppModule to register LoadController
- multi-stage Dockerfile now copies compiled dist/
- synced package-lock.json and installed redis types
- fixed iterator usage in storage.service

.dockerignore

@@ -0,0 +1,18 @@
+# ignore everything by default
+*
+
+# keep package manifests so we can install deps
+!package.json
+!package-lock.json
+
+# keep your Nest build config
+!nest-cli.json
+!tsconfig.json
+!tsconfig.build.json
+
+# keep source code
+!src/**
+
+# ignore node_modules and any pre-existing dist output
+node_modules
+dist

.gitlab-ci.yml

@@ -0,0 +1,36 @@
+include:
+  - project: "to-be-continuous/docker"
+    ref: "2.0.0"
+    file: "/templates/gitlab-ci-docker.yml"
+
+variables:
+  PROD_REF: "/^main$/"
+  AUTODEPLOY_TO_PROD: "true" # Always publish
+
+  DOCKER_REGISTRY_RELEASE_USER: kiliandeca
+  # DOCKER_REGISTRY_RELEASE_PASSWORD: # Defined in CI/CD Settings
+  DOCKER_RELEASE_IMAGE: docker.io/kiliandeca/excalidraw-storage-backend:latest
+
+stages:
+  - build
+  - package-build
+  - package-test
+  - publish
+
+.node-template:
+  image: node:14-alpine
+  before_script:
+    - npm ci --cache .npm --prefer-offline
+  cache:
+    key: npm-cache
+    paths:
+      - .npm-cache
+
+node-build:
+  stage: build
+  extends: .node-template
+  script:
+    - npm run build
+  artifacts:
+    paths:
+      - dist

Dockerfile

@@ -0,0 +1,14 @@
+FROM node:16-alpine
+
+WORKDIR /app
+
+COPY package*.json ./
+RUN npm ci --prod
+
+COPY ./dist ./dist
+
+USER node
+
+EXPOSE 8080
+
+ENTRYPOINT ["npm", "run", "start:prod"]

Dockerfile.override

@@ -0,0 +1,27 @@
+# --- Builder stage ----------------------------------------------------------
+    FROM node:16-alpine AS builder
+    WORKDIR /app
+    
+    # Install dev+prod dependencies and build TS
+    COPY package*.json ./
+    RUN npm install
+    COPY . .
+    RUN npm run build
+    
+    # --- Runtime stage ----------------------------------------------------------
+    FROM node:16-alpine
+    WORKDIR /app
+    
+    # Bring in package manifests so npm can locate the start script
+    COPY --from=builder /app/package*.json ./
+    
+    # Bring in only production deps and compiled output
+    COPY --from=builder /app/node_modules ./node_modules
+    COPY --from=builder /app/dist       ./dist
+    
+    # Drop to non-root user
+    USER node
+    
+    EXPOSE 8080
+    
+    ENTRYPOINT ["npm", "run", "start:prod"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Kilian Decaderincourt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,2 +1,25 @@
 # excalidraw-storage-backend
 
+This is a reimplementation of [excalidraw-json](https://github.com/excalidraw/excalidraw-json) suitable for self hosting you own instance of Excalidraw.
+
+It can be used with [kiliandeca/excalidraw-fork](https://gitlab.com/kiliandeca/excalidraw-fork)
+
+[DockerHub kiliandeca/excalidraw-storage-backend](https://hub.docker.com/r/kiliandeca/excalidraw-storage-backend)
+
+Feature:
+
+- Storing scenes: when you export as a link
+- Storing rooms: when you create a live collaboration
+- Storing images: when you export or do a live collaboration of a scene with images
+
+It use Keyv as a simple K/V store so you can use the database of your choice.
+
+## Environement Variables
+
+| Name            | Description                                                  | Default value    |
+| --------------- | ------------------------------------------------------------ | ---------------- |
+| `PORT`          | Server listening port                                        | 8080             |
+| `GLOBAL_PREFIX` | API global prefix for every routes                           | `/api/v2`        |
+| `STORAGE_URI`   | [Keyv](https://github.com/jaredwray/keyv) connection string, example: `redis://user:pass@localhost:6379`. Availabe Keyv storage adapter: redis, mongo, postgres and mysql  | `""` (in memory **non-persistent**) |
+| `LOG_LEVEL`     | Log level (`debug`, `verbose`, `log`, `warn`, `error`)       | `warn`           |
+| `BODY_LIMIT`    | Payload size limit for scenes or images                      | `50mb`           |

docker-compose.yml

@@ -0,0 +1,34 @@
+version: "3.8"
+
+services:
+  excalidraw: 
+    image: kiliandeca/excalidraw
+    healthcheck:
+      disable: true
+    ports:
+      - "80:80"
+    environment:
+      BACKEND_V2_GET_URL: http://localhost:8080/api/v2/scenes/
+      BACKEND_V2_POST_URL: http://localhost:8080/api/v2/scenes/
+      LIBRARY_URL: https://libraries.excalidraw.com
+      LIBRARY_BACKEND: https://us-central1-excalidraw-room-persistence.cloudfunctions.net/libraries
+      SOCKET_SERVER_URL: http://localhost:5000/
+      STORAGE_BACKEND: "http"
+      HTTP_STORAGE_BACKEND_URL: "http://localhost:8080/api/v2"
+
+  excalidraw-storage-backend:
+    build: .
+    ports:
+      - "8080:8080"
+    environment:
+      STORAGE_URI: redis://redis:6379
+
+  excalidraw-room:
+    image: excalidraw/excalidraw-room
+    ports:
+      - "5000:80"
+  
+  redis:
+    image: redis
+    ports:
+      - "6379:6379"

package.json

@@ -21,12 +21,21 @@
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
+    "@keyv/mongo": "^1.2.1",
+    "@keyv/mysql": "^1.1.5",
+    "@keyv/postgres": "^1.0.17",
+    "@keyv/redis": "^2.2.0",
+    "@keyv/sqlite": "^2.0.3",
     "@nestjs/common": "^8.0.0",
     "@nestjs/core": "^8.0.0",
     "@nestjs/platform-express": "^8.0.0",
+    "@types/keyv": "^3.1.3",
+    "keyv": "^4.0.4",
+    "nanoid": "^3.1.25",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
-    "rxjs": "^7.2.0"
+    "rxjs": "^7.2.0",
+    "redis": "^4.6.5"
   },
   "devDependencies": {
     "@nestjs/cli": "^8.0.0",

src/app.module.ts

@@ -1,12 +1,20 @@
 import { MiddlewareConsumer, Module } from '@nestjs/common';
 import { RawParserMiddleware } from './raw-parser.middleware';
 import { ScenesController } from './scenes/scenes.controller';
-import { MemoryService } from './storages/memory.service';
+import { LoadController } from './scenes/load.controller';
+import { StorageService } from './storage/storage.service';
+import { RoomsController } from './rooms/rooms.controller';
+import { FilesController } from './files/files.controller';
 
 @Module({
   imports: [],
-  controllers: [ScenesController],
+  controllers: [
-  providers: [MemoryService],
+    ScenesController,
+    LoadController,
+    RoomsController,
+    FilesController,
+  ],
+  providers: [StorageService],
 })
 export class AppModule {
   configure(consumer: MiddlewareConsumer) {

src/files/files.controller.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { FilesController } from './files.controller';
+
+describe('FilesController', () => {
+  let controller: FilesController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [FilesController],
+    }).compile();
+
+    controller = module.get<FilesController>(FilesController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

src/files/files.controller.ts

@@ -0,0 +1,49 @@
+import {
+  Body,
+  Controller,
+  Get,
+  Header,
+  Logger,
+  NotFoundException,
+  Param,
+  Put,
+  Res,
+} from '@nestjs/common';
+import { Response } from 'express';
+import { StorageNamespace, StorageService } from 'src/storage/storage.service';
+import { Readable } from 'stream';
+
+@Controller('files')
+export class FilesController {
+  private readonly logger = new Logger(FilesController.name);
+  namespace = StorageNamespace.FILES;
+
+  constructor(private storageService: StorageService) {}
+
+  @Get(':id')
+  @Header('content-type', 'application/octet-stream')
+  async findOne(@Param() params, @Res() res: Response): Promise<void> {
+    const data = await this.storageService.get(params.id, this.namespace);
+    this.logger.debug(`Get image ${params.id}`);
+
+    if (!data) {
+      throw new NotFoundException();
+    }
+
+    const stream = new Readable();
+    stream.push(data);
+    stream.push(null);
+    stream.pipe(res);
+  }
+
+  @Put(':id')
+  async create(@Param() params, @Body() payload: Buffer) {
+    const id = params.id;
+    await this.storageService.set(id, payload, this.namespace);
+    this.logger.debug(`Created image ${id}`);
+
+    return {
+      id,
+    };
+  }
+}

src/main.ts

@@ -1,12 +1,23 @@
+import { LogLevel } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
 import { AppModule } from './app.module';
 
+function isLogLevel(value: any): value is LogLevel {
+  return value in ['log', 'error', 'warn', 'debug', 'verbose'];
+}
+
 async function bootstrap() {
+  const logLevel = isLogLevel(process.env.LOG_LEVEL)
+    ? process.env.LOG_LEVEL
+    : 'log';
+
   const app = await NestFactory.create(AppModule, {
     cors: true,
+    logger: [logLevel],
   });
-  app.setGlobalPrefix('api/v2');
 
-  await app.listen(8080);
+  app.setGlobalPrefix(process.env.GLOBAL_PREFIX ?? '/api/v2');
+
+  await app.listen(process.env.PORT ?? 8080);
 }
 bootstrap();

src/raw-parser.middleware.ts

@@ -2,9 +2,12 @@ import { Injectable, NestMiddleware } from '@nestjs/common';
 import { raw } from 'express';
 import { hasBody } from 'type-is';
 
-// Excalidraw Front end doesn't send a Content Type Header 
+// Excalidraw Front end doesn't send a Content Type Header
 // so we tell raw parser to check if there is a body
-const rawParserMiddleware = raw({ type: hasBody });
+const rawParserMiddleware = raw({
+  type: hasBody,
+  limit: process.env.BODY_LIMIT ?? '50mb',
+});
 
 @Injectable()
 export class RawParserMiddleware implements NestMiddleware {

src/rooms/rooms.controller.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { RoomsController } from './rooms.controller';
+
+describe('RoomsController', () => {
+  let controller: RoomsController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [RoomsController],
+    }).compile();
+
+    controller = module.get<RoomsController>(RoomsController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

src/rooms/rooms.controller.ts

@@ -0,0 +1,49 @@
+import {
+  Body,
+  Controller,
+  Get,
+  Header,
+  Logger,
+  NotFoundException,
+  Param,
+  Put,
+  Res,
+} from '@nestjs/common';
+import { Response } from 'express';
+import { StorageNamespace, StorageService } from 'src/storage/storage.service';
+import { Readable } from 'stream';
+
+@Controller('rooms')
+export class RoomsController {
+  private readonly logger = new Logger(RoomsController.name);
+  namespace = StorageNamespace.ROOMS;
+
+  constructor(private storageService: StorageService) {}
+
+  @Get(':id')
+  @Header('content-type', 'application/octet-stream')
+  async findOne(@Param() params, @Res() res: Response): Promise<void> {
+    const data = await this.storageService.get(params.id, this.namespace);
+    this.logger.debug(`Get room ${params.id}`);
+
+    if (!data) {
+      throw new NotFoundException();
+    }
+
+    const stream = new Readable();
+    stream.push(data);
+    stream.push(null);
+    stream.pipe(res);
+  }
+
+  @Put(':id')
+  async create(@Param() params, @Body() payload: Buffer) {
+    const id = params.id;
+    await this.storageService.set(id, payload, this.namespace);
+    this.logger.debug(`Created room ${id}`);
+
+    return {
+      id,
+    };
+  }
+}

src/scenes/load.controller.ts

@@ -0,0 +1,60 @@
+import { Controller, Get, Logger, InternalServerErrorException } from '@nestjs/common';
+import { createClient } from 'redis';
+import { StorageNamespace } from 'src/storage/storage.service';
+
+@Controller('scenes')
+export class LoadController {
+  private readonly logger = new Logger(LoadController.name);
+
+  // Initialize Redis client once
+  private client = createClient({ url: process.env.STORAGE_URI });
+
+  constructor() {
+    this.client.on('error', (err) => {
+      this.logger.error('Redis client error', err);
+    });
+    this.client.connect().catch((err) => {
+      this.logger.error('Failed to connect Redis client', err);
+    });
+  }
+
+  @Get()
+  async listScenes() {
+    try {
+      const prefix = `${StorageNamespace.SCENES}:`;
+
+      // 1) list all keys
+      const keys = await this.client.keys(`${prefix}*`);
+      this.logger.debug(`Found ${keys.length} scene keys`);
+
+      // 2) pipeline a GET for each key
+      const pipeline = this.client.multi();
+      for (const k of keys) {
+        pipeline.get(k);
+      }
+      // pipeline.exec() on redis v4 returns an array of reply values
+      const raws = (await pipeline.exec()) as (string | null)[];
+
+      // 3) build summaries
+      const scenes = keys
+        .map((fullKey, idx) => {
+          const raw = raws[idx];
+          if (!raw) return null;
+          try {
+            const payload = JSON.parse(raw);
+            const id = fullKey.slice(prefix.length);
+            return { id, created: payload.created, updated: payload.updated };
+          } catch {
+            this.logger.warn(`Skipping invalid JSON at ${fullKey}`);
+            return null;
+          }
+        })
+        .filter((s) => s !== null);
+
+      return scenes;
+    } catch (err) {
+      this.logger.error('Error listing scenes', err as any);
+      throw new InternalServerErrorException('Failed to list scenes');
+    }
+  }
+}

src/scenes/scenes.controller.ts

@@ -3,22 +3,33 @@ import {
   Controller,
   Get,
   Header,
+  InternalServerErrorException,
+  Logger,
+  NotFoundException,
   Param,
   Post,
   Res,
 } from '@nestjs/common';
 import { Response } from 'express';
-import { MemoryService } from 'src/storages/memory.service';
+import { StorageNamespace, StorageService } from 'src/storage/storage.service';
-import { hash, hexadecimalToDecimal } from 'src/utils';
 import { Readable } from 'stream';
+import { customAlphabet } from 'nanoid';
 
-@Controller()
+@Controller('scenes')
 export class ScenesController {
-  constructor(private storageService: MemoryService) {}
+  private readonly logger = new Logger(ScenesController.name);
+  namespace = StorageNamespace.SCENES;
+
+  constructor(private storageService: StorageService) {}
   @Get(':id')
   @Header('content-type', 'application/octet-stream')
   async findOne(@Param() params, @Res() res: Response): Promise<void> {
-    const data = await this.storageService.load(params.id);
+    const data = await this.storageService.get(params.id, this.namespace);
+    this.logger.debug(`Get scene ${params.id}`);
+
+    if (!data) {
+      throw new NotFoundException();
+    }
 
     const stream = new Readable();
     stream.push(data);
@@ -28,15 +39,20 @@ export class ScenesController {
 
   @Post()
   async create(@Body() payload: Buffer) {
+    // Excalidraw front-end only support numeric id, we can't use nanoid default alphabet
+    const nanoid = customAlphabet('0123456789', 16);
+    const id = nanoid();
 
-    const drawingHash = hash(payload);
+    // Check for collision
-    const id = hexadecimalToDecimal(drawingHash);
+    if (await this.storageService.get(id, this.namespace)) {
+      throw new InternalServerErrorException();
+    }
 
-    await this.storageService.save(id, payload);
+    await this.storageService.set(id, payload, this.namespace);
+    this.logger.debug(`Created scene ${id}`);
 
     return {
       id,
-      data: `http://localhost:8080/api/v2/${id}`,
     };
   }
 }

src/storage/storage.service.spec.ts

@@ -1,15 +1,15 @@
 import { Test, TestingModule } from '@nestjs/testing';
-import { MemoryService } from './memory.service';
+import { StorageService } from './storage.service';
 
-describe('MemoryService', () => {
+describe('StorageService', () => {
-  let service: MemoryService;
+  let service: StorageService;
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
-      providers: [MemoryService],
+      providers: [StorageService],
     }).compile();
 
-    service = module.get<MemoryService>(MemoryService);
+    service = module.get<StorageService>(StorageService);
   });
 
   it('should be defined', () => {

src/storage/storage.service.ts

@@ -0,0 +1,43 @@
+import { Injectable, Logger } from '@nestjs/common';
+import * as Keyv from 'keyv';
+
+@Injectable()
+export class StorageService {
+  private readonly logger = new Logger(StorageService.name);
+  storagesMap = new Map<string, Keyv>();
+
+  constructor() {
+    const uri = process.env[`STORAGE_URI`];
+    if (!uri) {
+      this.logger.warn(
+        `STORAGE_URI is undefined, will use non persistant in memory storage`,
+      );
+    }
+
+    Object.keys(StorageNamespace).forEach((namespace) => {
+      const keyv = new Keyv({
+        uri,
+        namespace,
+      });
+      keyv.on('error', (err) =>
+        this.logger.error(`Connection Error for namespace ${namespace}`, err),
+      );
+      this.storagesMap.set(namespace, keyv);
+    });
+  }
+  get(key: string, namespace: StorageNamespace): Promise<Buffer> {
+    return this.storagesMap.get(namespace).get(key);
+  }
+  async has(key: string, namespace: StorageNamespace): Promise<boolean> {
+    return !!(await this.storagesMap.get(namespace).get(key));
+  }
+  set(key: string, value: Buffer, namespace: StorageNamespace): Promise<true> {
+    return this.storagesMap.get(namespace).set(key, value);
+  }
+}
+
+export enum StorageNamespace {
+  SCENES = 'SCENES',
+  ROOMS = 'ROOMS',
+  FILES = 'FILES',
+}

src/storages/memory.service.ts

@@ -1,17 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { StorageService } from './storageService';
-
-@Injectable()
-export class MemoryService implements StorageService {
-
-  scenesMap = new Map<string, Buffer>();
-
-  async save(id: string, data: Buffer): Promise<boolean> {
-    this.scenesMap.set(id, data);
-    return true;
-  }
-
-  async load(id: string): Promise<false | Buffer> {
-    return this.scenesMap.get(id);
-  }
-}

src/storages/storageService.ts

@@ -1,5 +0,0 @@
-export interface StorageService {
-  save(id: string, data: Buffer): Promise<boolean>;
-
-  load(id: string): Promise<Buffer | false>;
-}

src/utils.ts

@@ -1,12 +0,0 @@
-import { createHash } from 'crypto';
-
-export function hash(buffer): string {
-  return createHash(`sha256`).update(buffer).digest(`hex`);
-}
-
-// Copied from https://github.com/NMinhNguyen/excalidraw-json
-export function hexadecimalToDecimal(hexadecimal: string) {
-  // See https://stackoverflow.com/a/53751162
-  const bigInt = BigInt(`0x${hexadecimal}`);
-  return bigInt.toString(10);
-}