ci: add tbc node template

.dockerignore

@@ -1,18 +0,0 @@
-# ignore everything by default
-*
-
-# keep package manifests so we can install deps
-!package.json
-!package-lock.json
-
-# keep your Nest build config
-!nest-cli.json
-!tsconfig.json
-!tsconfig.build.json
-
-# keep source code
-!src/**
-
-# ignore node_modules and any pre-existing dist output
-node_modules
-dist

.gitlab-ci.yml

@@ -1,36 +1,4 @@
 include:
-  - project: "to-be-continuous/docker"
-    ref: "2.0.0"
-    file: "/templates/gitlab-ci-docker.yml"
-
-variables:
-  PROD_REF: "/^main$/"
-  AUTODEPLOY_TO_PROD: "true" # Always publish
-
-  DOCKER_REGISTRY_RELEASE_USER: kiliandeca
-  # DOCKER_REGISTRY_RELEASE_PASSWORD: # Defined in CI/CD Settings
-  DOCKER_RELEASE_IMAGE: docker.io/kiliandeca/excalidraw-storage-backend:latest
-
-stages:
-  - build
-  - package-build
-  - package-test
-  - publish
-
-.node-template:
-  image: node:14-alpine
-  before_script:
-    - npm ci --cache .npm --prefer-offline
-  cache:
-    key: npm-cache
-    paths:
-      - .npm-cache
-
-node-build:
-  stage: build
-  extends: .node-template
-  script:
-    - npm run build
-  artifacts:
-    paths:
-      - dist
+  - project: 'to-be-continuous/node'
+    ref: '1.2.0'
+    file: '/templates/gitlab-ci-node.yml'

Dockerfile

@@ -1,14 +0,0 @@
-FROM node:16-alpine
-
-WORKDIR /app
-
-COPY package*.json ./
-RUN npm ci --prod
-
-COPY ./dist ./dist
-
-USER node
-
-EXPOSE 8080
-
-ENTRYPOINT ["npm", "run", "start:prod"]

Dockerfile.override

@@ -1,27 +0,0 @@
-# --- Builder stage ----------------------------------------------------------
-    FROM node:16-alpine AS builder
-    WORKDIR /app
-    
-    # Install dev+prod dependencies and build TS
-    COPY package*.json ./
-    RUN npm install
-    COPY . .
-    RUN npm run build
-    
-    # --- Runtime stage ----------------------------------------------------------
-    FROM node:16-alpine
-    WORKDIR /app
-    
-    # Bring in package manifests so npm can locate the start script
-    COPY --from=builder /app/package*.json ./
-    
-    # Bring in only production deps and compiled output
-    COPY --from=builder /app/node_modules ./node_modules
-    COPY --from=builder /app/dist       ./dist
-    
-    # Drop to non-root user
-    USER node
-    
-    EXPOSE 8080
-    
-    ENTRYPOINT ["npm", "run", "start:prod"]

LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2022 Kilian Decaderincourt
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

README.md

@@ -1,25 +1,2 @@
 # excalidraw-storage-backend
 
-This is a reimplementation of [excalidraw-json](https://github.com/excalidraw/excalidraw-json) suitable for self hosting you own instance of Excalidraw.
-
-It can be used with [kiliandeca/excalidraw-fork](https://gitlab.com/kiliandeca/excalidraw-fork)
-
-[DockerHub kiliandeca/excalidraw-storage-backend](https://hub.docker.com/r/kiliandeca/excalidraw-storage-backend)
-
-Feature:
-
-- Storing scenes: when you export as a link
-- Storing rooms: when you create a live collaboration
-- Storing images: when you export or do a live collaboration of a scene with images
-
-It use Keyv as a simple K/V store so you can use the database of your choice.
-
-## Environement Variables
-
-| Name            | Description                                                  | Default value    |
-| --------------- | ------------------------------------------------------------ | ---------------- |
-| `PORT`          | Server listening port                                        | 8080             |
-| `GLOBAL_PREFIX` | API global prefix for every routes                           | `/api/v2`        |
-| `STORAGE_URI`   | [Keyv](https://github.com/jaredwray/keyv) connection string, example: `redis://user:pass@localhost:6379`. Availabe Keyv storage adapter: redis, mongo, postgres and mysql  | `""` (in memory **non-persistent**) |
-| `LOG_LEVEL`     | Log level (`debug`, `verbose`, `log`, `warn`, `error`)       | `warn`           |
-| `BODY_LIMIT`    | Payload size limit for scenes or images                      | `50mb`           |

docker-compose.yml

@@ -1,34 +0,0 @@
-version: "3.8"
-
-services:
-  excalidraw: 
-    image: kiliandeca/excalidraw
-    healthcheck:
-      disable: true
-    ports:
-      - "80:80"
-    environment:
-      BACKEND_V2_GET_URL: http://localhost:8080/api/v2/scenes/
-      BACKEND_V2_POST_URL: http://localhost:8080/api/v2/scenes/
-      LIBRARY_URL: https://libraries.excalidraw.com
-      LIBRARY_BACKEND: https://us-central1-excalidraw-room-persistence.cloudfunctions.net/libraries
-      SOCKET_SERVER_URL: http://localhost:5000/
-      STORAGE_BACKEND: "http"
-      HTTP_STORAGE_BACKEND_URL: "http://localhost:8080/api/v2"
-
-  excalidraw-storage-backend:
-    build: .
-    ports:
-      - "8080:8080"
-    environment:
-      STORAGE_URI: redis://redis:6379
-
-  excalidraw-room:
-    image: excalidraw/excalidraw-room
-    ports:
-      - "5000:80"
-  
-  redis:
-    image: redis
-    ports:
-      - "6379:6379"

package.json

@@ -21,21 +21,12 @@
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
-    "@keyv/mongo": "^1.2.1",
-    "@keyv/mysql": "^1.1.5",
-    "@keyv/postgres": "^1.0.17",
-    "@keyv/redis": "^2.2.0",
-    "@keyv/sqlite": "^2.0.3",
     "@nestjs/common": "^8.0.0",
     "@nestjs/core": "^8.0.0",
     "@nestjs/platform-express": "^8.0.0",
-    "@types/keyv": "^3.1.3",
-    "keyv": "^4.0.4",
-    "nanoid": "^3.1.25",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
-    "rxjs": "^7.2.0",
-    "redis": "^4.6.5"
+    "rxjs": "^7.2.0"
   },
   "devDependencies": {
     "@nestjs/cli": "^8.0.0",

src/app.module.ts

@@ -1,20 +1,12 @@
 import { MiddlewareConsumer, Module } from '@nestjs/common';
 import { RawParserMiddleware } from './raw-parser.middleware';
 import { ScenesController } from './scenes/scenes.controller';
-import { LoadController } from './scenes/load.controller';
-import { StorageService } from './storage/storage.service';
-import { RoomsController } from './rooms/rooms.controller';
-import { FilesController } from './files/files.controller';
+import { MemoryService } from './storages/memory.service';
 
 @Module({
   imports: [],
-  controllers: [
-    ScenesController,
-    LoadController,
-    RoomsController,
-    FilesController,
-  ],
-  providers: [StorageService],
+  controllers: [ScenesController],
+  providers: [MemoryService],
 })
 export class AppModule {
   configure(consumer: MiddlewareConsumer) {

src/files/files.controller.spec.ts

@@ -1,18 +0,0 @@
-import { Test, TestingModule } from '@nestjs/testing';
-import { FilesController } from './files.controller';
-
-describe('FilesController', () => {
-  let controller: FilesController;
-
-  beforeEach(async () => {
-    const module: TestingModule = await Test.createTestingModule({
-      controllers: [FilesController],
-    }).compile();
-
-    controller = module.get<FilesController>(FilesController);
-  });
-
-  it('should be defined', () => {
-    expect(controller).toBeDefined();
-  });
-});

src/files/files.controller.ts

@@ -1,49 +0,0 @@
-import {
-  Body,
-  Controller,
-  Get,
-  Header,
-  Logger,
-  NotFoundException,
-  Param,
-  Put,
-  Res,
-} from '@nestjs/common';
-import { Response } from 'express';
-import { StorageNamespace, StorageService } from 'src/storage/storage.service';
-import { Readable } from 'stream';
-
-@Controller('files')
-export class FilesController {
-  private readonly logger = new Logger(FilesController.name);
-  namespace = StorageNamespace.FILES;
-
-  constructor(private storageService: StorageService) {}
-
-  @Get(':id')
-  @Header('content-type', 'application/octet-stream')
-  async findOne(@Param() params, @Res() res: Response): Promise<void> {
-    const data = await this.storageService.get(params.id, this.namespace);
-    this.logger.debug(`Get image ${params.id}`);
-
-    if (!data) {
-      throw new NotFoundException();
-    }
-
-    const stream = new Readable();
-    stream.push(data);
-    stream.push(null);
-    stream.pipe(res);
-  }
-
-  @Put(':id')
-  async create(@Param() params, @Body() payload: Buffer) {
-    const id = params.id;
-    await this.storageService.set(id, payload, this.namespace);
-    this.logger.debug(`Created image ${id}`);
-
-    return {
-      id,
-    };
-  }
-}

src/main.ts

@@ -1,23 +1,12 @@
-import { LogLevel } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
 import { AppModule } from './app.module';
 
-function isLogLevel(value: any): value is LogLevel {
-  return value in ['log', 'error', 'warn', 'debug', 'verbose'];
-}
-
 async function bootstrap() {
-  const logLevel = isLogLevel(process.env.LOG_LEVEL)
-    ? process.env.LOG_LEVEL
-    : 'log';
-
   const app = await NestFactory.create(AppModule, {
     cors: true,
-    logger: [logLevel],
   });
+  app.setGlobalPrefix('api/v2');
 
-  app.setGlobalPrefix(process.env.GLOBAL_PREFIX ?? '/api/v2');
-
-  await app.listen(process.env.PORT ?? 8080);
+  await app.listen(8080);
 }
 bootstrap();

src/raw-parser.middleware.ts

@@ -4,10 +4,7 @@ import { hasBody } from 'type-is';
 
 // Excalidraw Front end doesn't send a Content Type Header 
 // so we tell raw parser to check if there is a body
-const rawParserMiddleware = raw({
-  type: hasBody,
-  limit: process.env.BODY_LIMIT ?? '50mb',
-});
+const rawParserMiddleware = raw({ type: hasBody });
 
 @Injectable()
 export class RawParserMiddleware implements NestMiddleware {

src/rooms/rooms.controller.spec.ts

@@ -1,18 +0,0 @@
-import { Test, TestingModule } from '@nestjs/testing';
-import { RoomsController } from './rooms.controller';
-
-describe('RoomsController', () => {
-  let controller: RoomsController;
-
-  beforeEach(async () => {
-    const module: TestingModule = await Test.createTestingModule({
-      controllers: [RoomsController],
-    }).compile();
-
-    controller = module.get<RoomsController>(RoomsController);
-  });
-
-  it('should be defined', () => {
-    expect(controller).toBeDefined();
-  });
-});

src/rooms/rooms.controller.ts

@@ -1,49 +0,0 @@
-import {
-  Body,
-  Controller,
-  Get,
-  Header,
-  Logger,
-  NotFoundException,
-  Param,
-  Put,
-  Res,
-} from '@nestjs/common';
-import { Response } from 'express';
-import { StorageNamespace, StorageService } from 'src/storage/storage.service';
-import { Readable } from 'stream';
-
-@Controller('rooms')
-export class RoomsController {
-  private readonly logger = new Logger(RoomsController.name);
-  namespace = StorageNamespace.ROOMS;
-
-  constructor(private storageService: StorageService) {}
-
-  @Get(':id')
-  @Header('content-type', 'application/octet-stream')
-  async findOne(@Param() params, @Res() res: Response): Promise<void> {
-    const data = await this.storageService.get(params.id, this.namespace);
-    this.logger.debug(`Get room ${params.id}`);
-
-    if (!data) {
-      throw new NotFoundException();
-    }
-
-    const stream = new Readable();
-    stream.push(data);
-    stream.push(null);
-    stream.pipe(res);
-  }
-
-  @Put(':id')
-  async create(@Param() params, @Body() payload: Buffer) {
-    const id = params.id;
-    await this.storageService.set(id, payload, this.namespace);
-    this.logger.debug(`Created room ${id}`);
-
-    return {
-      id,
-    };
-  }
-}

src/scenes/load.controller.ts

@@ -1,60 +0,0 @@
-import { Controller, Get, Logger, InternalServerErrorException } from '@nestjs/common';
-import { createClient } from 'redis';
-import { StorageNamespace } from 'src/storage/storage.service';
-
-@Controller('scenes')
-export class LoadController {
-  private readonly logger = new Logger(LoadController.name);
-
-  // Initialize Redis client once
-  private client = createClient({ url: process.env.STORAGE_URI });
-
-  constructor() {
-    this.client.on('error', (err) => {
-      this.logger.error('Redis client error', err);
-    });
-    this.client.connect().catch((err) => {
-      this.logger.error('Failed to connect Redis client', err);
-    });
-  }
-
-  @Get()
-  async listScenes() {
-    try {
-      const prefix = `${StorageNamespace.SCENES}:`;
-
-      // 1) list all keys
-      const keys = await this.client.keys(`${prefix}*`);
-      this.logger.debug(`Found ${keys.length} scene keys`);
-
-      // 2) pipeline a GET for each key
-      const pipeline = this.client.multi();
-      for (const k of keys) {
-        pipeline.get(k);
-      }
-      // pipeline.exec() on redis v4 returns an array of reply values
-      const raws = (await pipeline.exec()) as (string | null)[];
-
-      // 3) build summaries
-      const scenes = keys
-        .map((fullKey, idx) => {
-          const raw = raws[idx];
-          if (!raw) return null;
-          try {
-            const payload = JSON.parse(raw);
-            const id = fullKey.slice(prefix.length);
-            return { id, created: payload.created, updated: payload.updated };
-          } catch {
-            this.logger.warn(`Skipping invalid JSON at ${fullKey}`);
-            return null;
-          }
-        })
-        .filter((s) => s !== null);
-
-      return scenes;
-    } catch (err) {
-      this.logger.error('Error listing scenes', err as any);
-      throw new InternalServerErrorException('Failed to list scenes');
-    }
-  }
-}

src/scenes/scenes.controller.ts

@@ -3,33 +3,22 @@ import {
   Controller,
   Get,
   Header,
-  InternalServerErrorException,
-  Logger,
-  NotFoundException,
   Param,
   Post,
   Res,
 } from '@nestjs/common';
 import { Response } from 'express';
-import { StorageNamespace, StorageService } from 'src/storage/storage.service';
+import { MemoryService } from 'src/storages/memory.service';
+import { hash, hexadecimalToDecimal } from 'src/utils';
 import { Readable } from 'stream';
-import { customAlphabet } from 'nanoid';
 
-@Controller('scenes')
+@Controller()
 export class ScenesController {
-  private readonly logger = new Logger(ScenesController.name);
-  namespace = StorageNamespace.SCENES;
-
-  constructor(private storageService: StorageService) {}
+  constructor(private storageService: MemoryService) {}
   @Get(':id')
   @Header('content-type', 'application/octet-stream')
   async findOne(@Param() params, @Res() res: Response): Promise<void> {
-    const data = await this.storageService.get(params.id, this.namespace);
-    this.logger.debug(`Get scene ${params.id}`);
-
-    if (!data) {
-      throw new NotFoundException();
-    }
+    const data = await this.storageService.load(params.id);
 
     const stream = new Readable();
     stream.push(data);
@@ -39,20 +28,15 @@ export class ScenesController {
 
   @Post()
   async create(@Body() payload: Buffer) {
-    // Excalidraw front-end only support numeric id, we can't use nanoid default alphabet
-    const nanoid = customAlphabet('0123456789', 16);
-    const id = nanoid();
 
-    // Check for collision
-    if (await this.storageService.get(id, this.namespace)) {
-      throw new InternalServerErrorException();
-    }
+    const drawingHash = hash(payload);
+    const id = hexadecimalToDecimal(drawingHash);
 
-    await this.storageService.set(id, payload, this.namespace);
-    this.logger.debug(`Created scene ${id}`);
+    await this.storageService.save(id, payload);
 
     return {
       id,
+      data: `http://localhost:8080/api/v2/${id}`,
     };
   }
 }

src/storage/storage.service.ts

@@ -1,43 +0,0 @@
-import { Injectable, Logger } from '@nestjs/common';
-import * as Keyv from 'keyv';
-
-@Injectable()
-export class StorageService {
-  private readonly logger = new Logger(StorageService.name);
-  storagesMap = new Map<string, Keyv>();
-
-  constructor() {
-    const uri = process.env[`STORAGE_URI`];
-    if (!uri) {
-      this.logger.warn(
-        `STORAGE_URI is undefined, will use non persistant in memory storage`,
-      );
-    }
-
-    Object.keys(StorageNamespace).forEach((namespace) => {
-      const keyv = new Keyv({
-        uri,
-        namespace,
-      });
-      keyv.on('error', (err) =>
-        this.logger.error(`Connection Error for namespace ${namespace}`, err),
-      );
-      this.storagesMap.set(namespace, keyv);
-    });
-  }
-  get(key: string, namespace: StorageNamespace): Promise<Buffer> {
-    return this.storagesMap.get(namespace).get(key);
-  }
-  async has(key: string, namespace: StorageNamespace): Promise<boolean> {
-    return !!(await this.storagesMap.get(namespace).get(key));
-  }
-  set(key: string, value: Buffer, namespace: StorageNamespace): Promise<true> {
-    return this.storagesMap.get(namespace).set(key, value);
-  }
-}
-
-export enum StorageNamespace {
-  SCENES = 'SCENES',
-  ROOMS = 'ROOMS',
-  FILES = 'FILES',
-}

src/storages/memory.service.spec.ts

@@ -1,15 +1,15 @@
 import { Test, TestingModule } from '@nestjs/testing';
-import { StorageService } from './storage.service';
+import { MemoryService } from './memory.service';
 
-describe('StorageService', () => {
-  let service: StorageService;
+describe('MemoryService', () => {
+  let service: MemoryService;
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
-      providers: [StorageService],
+      providers: [MemoryService],
     }).compile();
 
-    service = module.get<StorageService>(StorageService);
+    service = module.get<MemoryService>(MemoryService);
   });
 
   it('should be defined', () => {

src/storages/memory.service.ts

@@ -0,0 +1,17 @@
+import { Injectable } from '@nestjs/common';
+import { StorageService } from './storageService';
+
+@Injectable()
+export class MemoryService implements StorageService {
+
+  scenesMap = new Map<string, Buffer>();
+
+  async save(id: string, data: Buffer): Promise<boolean> {
+    this.scenesMap.set(id, data);
+    return true;
+  }
+
+  async load(id: string): Promise<false | Buffer> {
+    return this.scenesMap.get(id);
+  }
+}

src/storages/storageService.ts

@@ -0,0 +1,5 @@
+export interface StorageService {
+  save(id: string, data: Buffer): Promise<boolean>;
+
+  load(id: string): Promise<Buffer | false>;
+}

src/utils.ts

@@ -0,0 +1,12 @@
+import { createHash } from 'crypto';
+
+export function hash(buffer): string {
+  return createHash(`sha256`).update(buffer).digest(`hex`);
+}
+
+// Copied from https://github.com/NMinhNguyen/excalidraw-json
+export function hexadecimalToDecimal(hexadecimal: string) {
+  // See https://stackoverflow.com/a/53751162
+  const bigInt = BigInt(`0x${hexadecimal}`);
+  return bigInt.toString(10);
+}