test level db

apps/server/package.json

@@ -6,6 +6,7 @@
   "exports": "./src/index.js",
   "dependencies": {
     "@googleapis/sheets": "^5.0.5",
+    "classic-level": "^3.0.0",
     "cookie": "^1.0.2",
     "cookie-parser": "^1.4.7",
     "cors": "^2.8.5",

apps/server/src/app.ts

@@ -243,47 +243,69 @@ export const startIntegrations = async () => {
  * @param {number} exitCode
  * @return {Promise<void>}
  */
-const shutdown = async (exitCode = 0) => {
-  consoleHighlight(`Ontime shutting down with code ${exitCode}`);
+const shutdown = (exitCode = 0) => {
+  consoleHighlight(`Ontime shutting down with code: ${exitCode}`);
+
+  // sync shutdowns
+  oscServer.shutdown();
+  socket.shutdown();
+  runtimeService.shutdown();
 
   // clear the restore file if it was a normal exit
   // 0 means it was a SIGNAL
   // 1 means crash -> keep the file
   // 2 means dev crash -> do nothing
   // 99 means there was a shutdown request from the UI
-  if (exitCode === 0 || exitCode === 99) {
-    await restoreService.clear();
-  }
 
-  expressServer?.close();
-  runtimeService.shutdown();
+  const pendingRestoreService = new Promise((resolve, _reject) => {
+    if (exitCode === 0 || exitCode === 99) {
+      restoreService.clear().then(resolve);
+    }
+    resolve;
+  });
+
+  const pendingExpressServer = new Promise((resolve, _reject) => {
+    expressServer?.close(resolve);
+  });
+
+  const pendingDataProvider = new Promise((resolve, _reject) => {
+    getDataProvider().shutdown().then(resolve);
+  });
+
+  Promise.all([pendingRestoreService, pendingExpressServer, pendingDataProvider]);
+
   logger.shutdown();
-  oscServer.shutdown();
-  socket.shutdown();
-  process.exit(exitCode);
+
+  expressServer?.close(() => {
+    getDataProvider()
+      .shutdown()
+      .then(() => {
+        process.exit(exitCode);
+      });
+  });
 };
 
 process.on('exit', (code) => consoleHighlight(`Ontime shutdown with code: ${code}`));
 
-process.on('unhandledRejection', async (error) => {
+process.on('unhandledRejection', (error) => {
   if (!isProduction && error instanceof Error && error.stack) {
     consoleError(error.stack);
   }
   generateCrashReport(error);
   logger.crash(LogOrigin.Server, `Uncaught rejection | ${error}`);
-  await shutdown(1);
+  shutdown(1);
 });
 
-process.on('uncaughtException', async (error) => {
+process.on('uncaughtException', (error) => {
   if (!isProduction && error instanceof Error && error.stack) {
     consoleError(error.stack);
   }
   generateCrashReport(error);
   logger.crash(LogOrigin.Server, `Uncaught exception | ${error}`);
-  await shutdown(1);
+  shutdown(1);
 });
 
 // register shutdown signals
-process.once('SIGHUP', async () => shutdown(0));
-process.once('SIGINT', async () => shutdown(0));
-process.once('SIGTERM', async () => shutdown(0));
+process.once('SIGHUP', () => shutdown(0));
+process.once('SIGINT', () => shutdown(0));
+process.once('SIGTERM', () => shutdown(0));

apps/server/src/classes/data-provider/DataProvider.ts

@@ -8,6 +8,7 @@ import {
   AutomationSettings,
   Rundown,
   ProjectRundowns,
+  LogOrigin,
 } from 'ontime-types';
 
 import type { Low } from 'lowdb';
@@ -23,6 +24,18 @@ type ReadonlyPromise<T> = Promise<Readonly<T>>;
 
 let db = {} as Low<DatabaseModel>;
 
+import { publicDir } from '../../setup/index.js';
+import { ClassicLevel } from 'classic-level';
+import { logger } from '../Logger.js';
+
+const main_db = new ClassicLevel<keyof DatabaseModel, any>(`${publicDir.projectsDir}/db`, {
+  valueEncoding: 'json',
+});
+
+const rundown_db = main_db.sublevel<string, Rundown>('rundowns', {
+  valueEncoding: 'json',
+});
+
 /**
  * Initialises the JSON adapter to persist data to a file
  */
@@ -31,6 +44,19 @@ export async function initPersistence(filePath: string, fallbackData: DatabaseMo
   DEV: shouldCrashDev(!isPath(filePath), 'initPersistence should be called with a path');
   const newDb = await JSONFilePreset<DatabaseModel>(filePath, fallbackData);
 
+  const { project, settings, viewSettings, urlPresets, customFields, automation, rundowns } = fallbackData;
+  await main_db.open();
+  await main_db.put('project', project);
+  await main_db.put('settings', settings);
+  await main_db.put('viewSettings', viewSettings);
+  await main_db.put('urlPresets', urlPresets);
+  await main_db.put('customFields', customFields);
+  await main_db.put('automation', automation);
+
+  Object.entries(rundowns).forEach(([key, rundown]) => {
+    rundown_db.put(key, rundown);
+  });
+
   // Read the database to initialize it
   newDb.data = fallbackData;
   await newDb.write();
@@ -60,6 +86,7 @@ export function getDataProvider() {
     setAutomation,
     getRundown,
     mergeIntoData,
+    shutdown,
   };
 }
 
@@ -68,13 +95,13 @@ function getData(): Readonly<DatabaseModel> {
 }
 
 async function setProjectData(newData: Partial<ProjectData>): ReadonlyPromise<ProjectData> {
-  db.data.project = { ...db.data.project, ...newData };
-  await persist();
-  return db.data.project;
+  const newProjectData = { ...getProjectData(), ...newData };
+  await main_db.put('project', newProjectData);
+  return newProjectData;
 }
 
-function getProjectData(): Readonly<ProjectData> {
-  return db.data.project;
+function getProjectData(): ProjectData {
+  return main_db.getSync('project') as ProjectData;
 }
 
 async function setCustomFields(newData: CustomFields): ReadonlyPromise<CustomFields> {
@@ -97,8 +124,8 @@ async function mergeRundown(
   return { rundowns: db.data.rundowns, customFields: db.data.customFields };
 }
 
-function getCustomFields(): Readonly<CustomFields> {
-  return db.data.customFields;
+function getCustomFields(): CustomFields {
+  return main_db.getSync('customFields') as CustomFields;
 }
 
 async function setRundown(rundownKey: string, newData: Rundown): ReadonlyPromise<Rundown> {
@@ -107,8 +134,8 @@ async function setRundown(rundownKey: string, newData: Rundown): ReadonlyPromise
   return db.data.rundowns[rundownKey];
 }
 
-function getSettings(): Readonly<Settings> {
-  return db.data.settings;
+function getSettings(): Settings {
+  return main_db.getSync('settings') as Settings;
 }
 
 async function setSettings(newData: Settings): ReadonlyPromise<Settings> {
@@ -117,8 +144,8 @@ async function setSettings(newData: Settings): ReadonlyPromise<Settings> {
   return db.data.settings;
 }
 
-function getUrlPresets(): Readonly<URLPreset[]> {
-  return db.data.urlPresets;
+function getUrlPresets(): URLPreset[] {
+  return main_db.getSync('urlPresets') as URLPreset[];
 }
 
 async function setUrlPresets(newData: URLPreset[]): ReadonlyPromise<URLPreset[]> {
@@ -127,8 +154,8 @@ async function setUrlPresets(newData: URLPreset[]): ReadonlyPromise<URLPreset[]>
   return db.data.urlPresets;
 }
 
-function getViewSettings(): Readonly<ViewSettings> {
-  return db.data.viewSettings;
+function getViewSettings(): ViewSettings {
+  return main_db.getSync('viewSettings');
 }
 
 async function setViewSettings(newData: ViewSettings): ReadonlyPromise<ViewSettings> {
@@ -137,8 +164,10 @@ async function setViewSettings(newData: ViewSettings): ReadonlyPromise<ViewSetti
   return db.data.viewSettings;
 }
 
-function getAutomation(): Readonly<AutomationSettings> {
-  return db.data.automation;
+function getAutomation(): AutomationSettings {
+  const automation = main_db.getSync('automation');
+  if (!automation) throw new Error('Failed to load automation from db');
+  return automation;
 }
 
 async function setAutomation(newData: AutomationSettings): ReadonlyPromise<AutomationSettings> {
@@ -147,9 +176,10 @@ async function setAutomation(newData: AutomationSettings): ReadonlyPromise<Autom
   return db.data.automation;
 }
 
-function getRundown(): Readonly<Rundown> {
-  const firstRundown = Object.keys(db.data.rundowns)[0];
-  return db.data.rundowns[firstRundown];
+function getRundown(): Rundown {
+  const rundown = rundown_db.getSync('default');
+  if (!rundown) throw new Error('Failed to load rundown from db');
+  return rundown;
 }
 
 async function mergeIntoData(newData: Partial<DatabaseModel>): ReadonlyPromise<DatabaseModel> {
@@ -173,3 +203,8 @@ async function persist() {
   if (isTest) return;
   await db.write();
 }
+
+async function shutdown() {
+  logger.info(LogOrigin.Server, 'Closing DB');
+  await main_db.close();
+}

apps/server/src/classes/data-provider/levelDb.utils.ts

@@ -0,0 +1,24 @@
+export async function batchPutObject(obj: object, db) {
+  await db.batch(
+    Object.entries(obj).map(([key, value]) => {
+      return value === null ? { type: 'del', key } : { type: 'put', key, value };
+    }),
+  );
+}
+
+//   await projectDb.batch([
+//     { type: 'put', key: 'title', value: project.title },
+//     { type: 'put', key: 'description', value: project.description },
+//     { type: 'put', key: 'publicUrl', value: project.publicUrl },
+//     { type: 'put', key: 'publicInfo', value: project.publicInfo },
+//     { type: 'put', key: 'backstageUrl', value: project.backstageUrl },
+//     { type: 'put', key: 'backstageInfo', value: project.backstageInfo },
+//     project.backstageInfo
+//       ? { type: 'put', key: 'projectLogo', value: project.projectLogo }
+//       : { type: 'del', key: 'projectLogo' },
+//   ]);
+
+//   // await levelDb.put('project', project);
+
+//   console.log('level',levelDb.getSync('project'));
+//   console.log('project',projectDb.getSync('title'));

pnpm-lock.yaml

@@ -288,6 +288,9 @@ importers:
       '@googleapis/sheets':
         specifier: ^5.0.5
         version: 5.0.5
+      classic-level:
+        specifier: ^3.0.0
+        version: 3.0.0
       cookie:
         specifier: ^1.0.2
         version: 1.0.2
@@ -2272,6 +2275,10 @@ packages:
     resolution: {integrity: sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==}
     deprecated: Use your platform's native atob() and btoa() methods instead
 
+  abstract-level@3.1.0:
+    resolution: {integrity: sha512-j2e+TsAxy7Ri+0h7dJqwasymgt0zHBWX4+nMk3XatyuqgHfdstBJ9wsMfbiGwE1O+QovRyPcVAqcViMYdyPaaw==}
+    engines: {node: '>=18'}
+
   accepts@1.3.8:
     resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==}
     engines: {node: '>= 0.6'}
@@ -2488,6 +2495,9 @@ packages:
   buffer@5.7.1:
     resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
 
+  buffer@6.0.3:
+    resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
+
   builder-util-runtime@9.2.4:
     resolution: {integrity: sha512-upp+biKpN/XZMLim7aguUyW8s0FUpDvOtK6sbanMFDAMBzpHDqdhgVYm6zc9HJ6nWo7u2Lxk60i2M6Jd3aiNrA==}
     engines: {node: '>=12.0.0'}
@@ -2584,6 +2594,10 @@ packages:
     resolution: {integrity: sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==}
     engines: {node: '>=8'}
 
+  classic-level@3.0.0:
+    resolution: {integrity: sha512-yGy8j8LjPbN0Bh3+ygmyYvrmskVita92pD/zCoalfcC9XxZj6iDtZTAnz+ot7GG8p9KLTG+MZ84tSA4AhkgVZQ==}
+    engines: {node: '>=18'}
+
   cli-truncate@2.1.0:
     resolution: {integrity: sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg==}
     engines: {node: '>=8'}
@@ -3570,6 +3584,10 @@ packages:
     resolution: {integrity: sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==}
     engines: {node: '>= 0.4'}
 
+  is-buffer@2.0.5:
+    resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==}
+    engines: {node: '>=4'}
+
   is-callable@1.2.7:
     resolution: {integrity: sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==}
     engines: {node: '>= 0.4'}
@@ -3738,6 +3756,14 @@ packages:
     resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==}
     engines: {node: '>= 0.6.3'}
 
+  level-supports@6.2.0:
+    resolution: {integrity: sha512-QNxVXP0IRnBmMsJIh+sb2kwNCYcKciQZJEt+L1hPCHrKNELllXhvrlClVHXBYZVT+a7aTSM6StgNXdAldoab3w==}
+    engines: {node: '>=16'}
+
+  level-transcoder@1.0.1:
+    resolution: {integrity: sha512-t7bFwFtsQeD8cl8NIoQ2iwxA0CL/9IFw7/9gAjOonH0PWTTiRfY7Hq+Ejbsxh86tXobDQ6IOiddjNYIfOBs06w==}
+    engines: {node: '>=12'}
+
   levn@0.4.1:
     resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==}
     engines: {node: '>= 0.8.0'}
@@ -3821,6 +3847,10 @@ packages:
     resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==}
     engines: {node: '>= 0.4'}
 
+  maybe-combine-errors@1.0.0:
+    resolution: {integrity: sha512-eefp6IduNPT6fVdwPp+1NgD0PML1NU5P6j1Mj5nz1nidX8/sWY7119WL8vTAHgqfsY74TzW0w1XPgdYEKkGZ5A==}
+    engines: {node: '>=10'}
+
   media-typer@0.3.0:
     resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}
     engines: {node: '>= 0.6'}
@@ -3917,6 +3947,10 @@ packages:
     engines: {node: '>=10'}
     hasBin: true
 
+  module-error@1.0.2:
+    resolution: {integrity: sha512-0yuvsqSCv8LbaOKhnsQ/T5JhyFlCYLPXK3U2sgV10zoKQwzs/MyfuQUOZQ1V/6OCOJsK/TRgNVrPuPDqtdMFtA==}
+    engines: {node: '>=10'}
+
   ms@2.0.0:
     resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==}
 
@@ -3937,6 +3971,9 @@ packages:
     engines: {node: ^18 || >=20}
     hasBin: true
 
+  napi-macros@2.2.2:
+    resolution: {integrity: sha512-hmEVtAGYzVQpCKdbQea4skABsdXW4RUh5t5mJ2zzqowJS2OyXZTU1KhDVFhx+NlWZ4ap9mqR9TcDO3LTTttd+g==}
+
   natural-compare@1.4.0:
     resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==}
 
@@ -3959,6 +3996,10 @@ packages:
       encoding:
         optional: true
 
+  node-gyp-build@4.8.4:
+    resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==}
+    hasBin: true
+
   node-releases@2.0.14:
     resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==}
 
@@ -7115,6 +7156,15 @@ snapshots:
   abab@2.0.6:
     optional: true
 
+  abstract-level@3.1.0:
+    dependencies:
+      buffer: 6.0.3
+      is-buffer: 2.0.5
+      level-supports: 6.2.0
+      level-transcoder: 1.0.1
+      maybe-combine-errors: 1.0.0
+      module-error: 1.0.2
+
   accepts@1.3.8:
     dependencies:
       mime-types: 2.1.35
@@ -7396,6 +7446,11 @@ snapshots:
       base64-js: 1.5.1
       ieee754: 1.2.1
 
+  buffer@6.0.3:
+    dependencies:
+      base64-js: 1.5.1
+      ieee754: 1.2.1
+
   builder-util-runtime@9.2.4:
     dependencies:
       debug: 4.3.7
@@ -7526,6 +7581,13 @@ snapshots:
 
   ci-info@3.9.0: {}
 
+  classic-level@3.0.0:
+    dependencies:
+      abstract-level: 3.1.0
+      module-error: 1.0.2
+      napi-macros: 2.2.2
+      node-gyp-build: 4.8.4
+
   cli-truncate@2.1.0:
     dependencies:
       slice-ansi: 3.0.0
@@ -8784,6 +8846,8 @@ snapshots:
       call-bind: 1.0.2
       has-tostringtag: 1.0.2
 
+  is-buffer@2.0.5: {}
+
   is-callable@1.2.7: {}
 
   is-ci@3.0.1:
@@ -8970,6 +9034,13 @@ snapshots:
     dependencies:
       readable-stream: 2.3.8
 
+  level-supports@6.2.0: {}
+
+  level-transcoder@1.0.1:
+    dependencies:
+      buffer: 6.0.3
+      module-error: 1.0.2
+
   levn@0.4.1:
     dependencies:
       prelude-ls: 1.2.1
@@ -9040,6 +9111,8 @@ snapshots:
 
   math-intrinsics@1.1.0: {}
 
+  maybe-combine-errors@1.0.0: {}
+
   media-typer@0.3.0: {}
 
   merge-descriptors@1.0.3: {}
@@ -9108,6 +9181,8 @@ snapshots:
 
   mkdirp@1.0.4: {}
 
+  module-error@1.0.2: {}
+
   ms@2.0.0: {}
 
   ms@2.1.3: {}
@@ -9126,6 +9201,8 @@ snapshots:
 
   nanoid@5.0.7: {}
 
+  napi-macros@2.2.2: {}
+
   natural-compare@1.4.0: {}
 
   negotiator@0.6.3: {}
@@ -9142,6 +9219,8 @@ snapshots:
     dependencies:
       whatwg-url: 5.0.0
 
+  node-gyp-build@4.8.4: {}
+
   node-releases@2.0.14: {}
 
   normalize-path@3.0.0: {}