test level db

apps/server/package.json

@@ -6,6 +6,7 @@
   "exports": "./src/index.js",
   "dependencies": {
     "@googleapis/sheets": "^5.0.5",
+    "classic-level": "^3.0.0",
     "cookie": "^1.0.2",
     "cookie-parser": "^1.4.7",
     "cors": "^2.8.5",

apps/server/src/app.ts

@@ -243,47 +243,69 @@ export const startIntegrations = async () => {
  * @param {number} exitCode
  * @return {Promise<void>}
  */
-const shutdown = async (exitCode = 0) => {
-  consoleHighlight(`Ontime shutting down with code ${exitCode}`);
+const shutdown = (exitCode = 0) => {
+  consoleHighlight(`Ontime shutting down with code: ${exitCode}`);
+
+  // sync shutdowns
+  oscServer.shutdown();
+  socket.shutdown();
+  runtimeService.shutdown();
 
   // clear the restore file if it was a normal exit
   // 0 means it was a SIGNAL
   // 1 means crash -> keep the file
   // 2 means dev crash -> do nothing
   // 99 means there was a shutdown request from the UI
-  if (exitCode === 0 || exitCode === 99) {
-    await restoreService.clear();
-  }
 
-  expressServer?.close();
-  runtimeService.shutdown();
+  const pendingRestoreService = new Promise((resolve, _reject) => {
+    if (exitCode === 0 || exitCode === 99) {
+      restoreService.clear().then(resolve);
+    }
+    resolve;
+  });
+
+  const pendingExpressServer = new Promise((resolve, _reject) => {
+    expressServer?.close(resolve);
+  });
+
+  const pendingDataProvider = new Promise((resolve, _reject) => {
+    getDataProvider().shutdown().then(resolve);
+  });
+
+  Promise.all([pendingRestoreService, pendingExpressServer, pendingDataProvider]);
+
   logger.shutdown();
-  oscServer.shutdown();
-  socket.shutdown();
-  process.exit(exitCode);
+
+  expressServer?.close(() => {
+    getDataProvider()
+      .shutdown()
+      .then(() => {
+        process.exit(exitCode);
+      });
+  });
 };
 
 process.on('exit', (code) => consoleHighlight(`Ontime shutdown with code: ${code}`));
 
-process.on('unhandledRejection', async (error) => {
+process.on('unhandledRejection', (error) => {
   if (!isProduction && error instanceof Error && error.stack) {
     consoleError(error.stack);
   }
   generateCrashReport(error);
   logger.crash(LogOrigin.Server, `Uncaught rejection | ${error}`);
-  await shutdown(1);
+  shutdown(1);
 });
 
-process.on('uncaughtException', async (error) => {
+process.on('uncaughtException', (error) => {
   if (!isProduction && error instanceof Error && error.stack) {
     consoleError(error.stack);
   }
   generateCrashReport(error);
   logger.crash(LogOrigin.Server, `Uncaught exception | ${error}`);
-  await shutdown(1);
+  shutdown(1);
 });
 
 // register shutdown signals
-process.once('SIGHUP', async () => shutdown(0));
-process.once('SIGINT', async () => shutdown(0));
-process.once('SIGTERM', async () => shutdown(0));
+process.once('SIGHUP', () => shutdown(0));
+process.once('SIGINT', () => shutdown(0));
+process.once('SIGTERM', () => shutdown(0));

apps/server/src/classes/data-provider/DataProvider.ts

@@ -8,6 +8,7 @@ import {
   AutomationSettings,
   Rundown,
   ProjectRundowns,
+  LogOrigin,
 } from 'ontime-types';
 
 import type { Low } from 'lowdb';
@@ -23,6 +24,18 @@ type ReadonlyPromise<T> = Promise<Readonly<T>>;
 
 let db = {} as Low<DatabaseModel>;
 
+import { publicDir } from '../../setup/index.js';
+import { ClassicLevel } from 'classic-level';
+import { logger } from '../Logger.js';
+
+const main_db = new ClassicLevel<keyof DatabaseModel, any>(`${publicDir.projectsDir}/db`, {
+  valueEncoding: 'json',
+});
+
+const rundown_db = main_db.sublevel<string, Rundown>('rundowns', {
+  valueEncoding: 'json',
+});
+
 /**
  * Initialises the JSON adapter to persist data to a file
  */
@@ -31,6 +44,19 @@ export async function initPersistence(filePath: string, fallbackData: DatabaseMo
   DEV: shouldCrashDev(!isPath(filePath), 'initPersistence should be called with a path');
   const newDb = await JSONFilePreset<DatabaseModel>(filePath, fallbackData);
 
+  const { project, settings, viewSettings, urlPresets, customFields, automation, rundowns } = fallbackData;
+  await main_db.open();
+  await main_db.put('project', project);
+  await main_db.put('settings', settings);
+  await main_db.put('viewSettings', viewSettings);
+  await main_db.put('urlPresets', urlPresets);
+  await main_db.put('customFields', customFields);
+  await main_db.put('automation', automation);
+
+  Object.entries(rundowns).forEach(([key, rundown]) => {
+    rundown_db.put(key, rundown);
+  });
+
   // Read the database to initialize it
   newDb.data = fallbackData;
   await newDb.write();
@@ -60,6 +86,7 @@ export function getDataProvider() {
     setAutomation,
     getRundown,
     mergeIntoData,
+    shutdown,
   };
 }
 
@@ -68,13 +95,13 @@ function getData(): Readonly<DatabaseModel> {
 }
 
 async function setProjectData(newData: Partial<ProjectData>): ReadonlyPromise<ProjectData> {
-  db.data.project = { ...db.data.project, ...newData };
-  await persist();
-  return db.data.project;
+  const newProjectData = { ...getProjectData(), ...newData };
+  await main_db.put('project', newProjectData);
+  return newProjectData;
 }
 
-function getProjectData(): Readonly<ProjectData> {
-  return db.data.project;
+function getProjectData(): ProjectData {
+  return main_db.getSync('project') as ProjectData;
 }
 
 async function setCustomFields(newData: CustomFields): ReadonlyPromise<CustomFields> {
@@ -97,8 +124,8 @@ async function mergeRundown(
   return { rundowns: db.data.rundowns, customFields: db.data.customFields };
 }
 
-function getCustomFields(): Readonly<CustomFields> {
-  return db.data.customFields;
+function getCustomFields(): CustomFields {
+  return main_db.getSync('customFields') as CustomFields;
 }
 
 async function setRundown(rundownKey: string, newData: Rundown): ReadonlyPromise<Rundown> {
@@ -107,8 +134,8 @@ async function setRundown(rundownKey: string, newData: Rundown): ReadonlyPromise
   return db.data.rundowns[rundownKey];
 }
 
-function getSettings(): Readonly<Settings> {
-  return db.data.settings;
+function getSettings(): Settings {
+  return main_db.getSync('settings') as Settings;
 }
 
 async function setSettings(newData: Settings): ReadonlyPromise<Settings> {
@@ -117,8 +144,8 @@ async function setSettings(newData: Settings): ReadonlyPromise<Settings> {
   return db.data.settings;
 }
 
-function getUrlPresets(): Readonly<URLPreset[]> {
-  return db.data.urlPresets;
+function getUrlPresets(): URLPreset[] {
+  return main_db.getSync('urlPresets') as URLPreset[];
 }
 
 async function setUrlPresets(newData: URLPreset[]): ReadonlyPromise<URLPreset[]> {
@@ -127,8 +154,8 @@ async function setUrlPresets(newData: URLPreset[]): ReadonlyPromise<URLPreset[]>
   return db.data.urlPresets;
 }
 
-function getViewSettings(): Readonly<ViewSettings> {
-  return db.data.viewSettings;
+function getViewSettings(): ViewSettings {
+  return main_db.getSync('viewSettings');
 }
 
 async function setViewSettings(newData: ViewSettings): ReadonlyPromise<ViewSettings> {
@@ -137,8 +164,10 @@ async function setViewSettings(newData: ViewSettings): ReadonlyPromise<ViewSetti
   return db.data.viewSettings;
 }
 
-function getAutomation(): Readonly<AutomationSettings> {
-  return db.data.automation;
+function getAutomation(): AutomationSettings {
+  const automation = main_db.getSync('automation');
+  if (!automation) throw new Error('Failed to load automation from db');
+  return automation;
 }
 
 async function setAutomation(newData: AutomationSettings): ReadonlyPromise<AutomationSettings> {
@@ -147,9 +176,10 @@ async function setAutomation(newData: AutomationSettings): ReadonlyPromise<Autom
   return db.data.automation;
 }
 
-function getRundown(): Readonly<Rundown> {
-  const firstRundown = Object.keys(db.data.rundowns)[0];
-  return db.data.rundowns[firstRundown];
+function getRundown(): Rundown {
+  const rundown = rundown_db.getSync('default');
+  if (!rundown) throw new Error('Failed to load rundown from db');
+  return rundown;
 }
 
 async function mergeIntoData(newData: Partial<DatabaseModel>): ReadonlyPromise<DatabaseModel> {
@@ -173,3 +203,8 @@ async function persist() {
   if (isTest) return;
   await db.write();
 }
+
+async function shutdown() {
+  logger.info(LogOrigin.Server, 'Closing DB');
+  await main_db.close();
+}

apps/server/src/classes/data-provider/levelDb.utils.ts

@@ -0,0 +1,24 @@
+export async function batchPutObject(obj: object, db) {
+  await db.batch(
+    Object.entries(obj).map(([key, value]) => {
+      return value === null ? { type: 'del', key } : { type: 'put', key, value };
+    }),
+  );
+}
+
+//   await projectDb.batch([
+//     { type: 'put', key: 'title', value: project.title },
+//     { type: 'put', key: 'description', value: project.description },
+//     { type: 'put', key: 'publicUrl', value: project.publicUrl },
+//     { type: 'put', key: 'publicInfo', value: project.publicInfo },
+//     { type: 'put', key: 'backstageUrl', value: project.backstageUrl },
+//     { type: 'put', key: 'backstageInfo', value: project.backstageInfo },
+//     project.backstageInfo
+//       ? { type: 'put', key: 'projectLogo', value: project.projectLogo }
+//       : { type: 'del', key: 'projectLogo' },
+//   ]);
+
+//   // await levelDb.put('project', project);
+
+//   console.log('level',levelDb.getSync('project'));
+//   console.log('project',projectDb.getSync('title'));