From 6348defd42b98f757e20079e16f8119909345823 Mon Sep 17 00:00:00 2001 From: arc-alex Date: Wed, 23 Apr 2025 08:27:52 +0200 Subject: [PATCH] test level db --- apps/server/package.json | 1 + apps/server/src/app.ts | 56 +++++++++---- .../src/classes/data-provider/DataProvider.ts | 71 ++++++++++++----- .../classes/data-provider/levelDb.utils.ts | 24 ++++++ pnpm-lock.yaml | 79 +++++++++++++++++++ 5 files changed, 196 insertions(+), 35 deletions(-) create mode 100644 apps/server/src/classes/data-provider/levelDb.utils.ts diff --git a/apps/server/package.json b/apps/server/package.json index 6f50b0606..55f7db263 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -6,6 +6,7 @@ "exports": "./src/index.js", "dependencies": { "@googleapis/sheets": "^5.0.5", + "classic-level": "^3.0.0", "cookie": "^1.0.2", "cookie-parser": "^1.4.7", "cors": "^2.8.5", diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 954ab6c32..48f43f385 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -243,47 +243,69 @@ export const startIntegrations = async () => { * @param {number} exitCode * @return {Promise} */ -const shutdown = async (exitCode = 0) => { - consoleHighlight(`Ontime shutting down with code ${exitCode}`); +const shutdown = (exitCode = 0) => { + consoleHighlight(`Ontime shutting down with code: ${exitCode}`); + + // sync shutdowns + oscServer.shutdown(); + socket.shutdown(); + runtimeService.shutdown(); // clear the restore file if it was a normal exit // 0 means it was a SIGNAL // 1 means crash -> keep the file // 2 means dev crash -> do nothing // 99 means there was a shutdown request from the UI - if (exitCode === 0 || exitCode === 99) { - await restoreService.clear(); - } - expressServer?.close(); - runtimeService.shutdown(); + const pendingRestoreService = new Promise((resolve, _reject) => { + if (exitCode === 0 || exitCode === 99) { + restoreService.clear().then(resolve); + } + resolve; + }); + + const pendingExpressServer = new Promise((resolve, _reject) => { + expressServer?.close(resolve); + }); + + const pendingDataProvider = new Promise((resolve, _reject) => { + getDataProvider().shutdown().then(resolve); + }); + + Promise.all([pendingRestoreService, pendingExpressServer, pendingDataProvider]); + logger.shutdown(); - oscServer.shutdown(); - socket.shutdown(); - process.exit(exitCode); + + expressServer?.close(() => { + getDataProvider() + .shutdown() + .then(() => { + process.exit(exitCode); + }); + }); }; process.on('exit', (code) => consoleHighlight(`Ontime shutdown with code: ${code}`)); -process.on('unhandledRejection', async (error) => { +process.on('unhandledRejection', (error) => { if (!isProduction && error instanceof Error && error.stack) { consoleError(error.stack); } generateCrashReport(error); logger.crash(LogOrigin.Server, `Uncaught rejection | ${error}`); - await shutdown(1); + shutdown(1); }); -process.on('uncaughtException', async (error) => { +process.on('uncaughtException', (error) => { if (!isProduction && error instanceof Error && error.stack) { consoleError(error.stack); } generateCrashReport(error); logger.crash(LogOrigin.Server, `Uncaught exception | ${error}`); - await shutdown(1); + shutdown(1); }); // register shutdown signals -process.once('SIGHUP', async () => shutdown(0)); -process.once('SIGINT', async () => shutdown(0)); -process.once('SIGTERM', async () => shutdown(0)); +process.once('SIGHUP', () => shutdown(0)); +process.once('SIGINT', () => shutdown(0)); +process.once('SIGTERM', () => shutdown(0)); diff --git a/apps/server/src/classes/data-provider/DataProvider.ts b/apps/server/src/classes/data-provider/DataProvider.ts index 4793a123d..fbf7e6a3b 100644 --- a/apps/server/src/classes/data-provider/DataProvider.ts +++ b/apps/server/src/classes/data-provider/DataProvider.ts @@ -8,6 +8,7 @@ import { AutomationSettings, Rundown, ProjectRundowns, + LogOrigin, } from 'ontime-types'; import type { Low } from 'lowdb'; @@ -23,6 +24,18 @@ type ReadonlyPromise = Promise>; let db = {} as Low; +import { publicDir } from '../../setup/index.js'; +import { ClassicLevel } from 'classic-level'; +import { logger } from '../Logger.js'; + +const main_db = new ClassicLevel(`${publicDir.projectsDir}/db`, { + valueEncoding: 'json', +}); + +const rundown_db = main_db.sublevel('rundowns', { + valueEncoding: 'json', +}); + /** * Initialises the JSON adapter to persist data to a file */ @@ -31,6 +44,19 @@ export async function initPersistence(filePath: string, fallbackData: DatabaseMo DEV: shouldCrashDev(!isPath(filePath), 'initPersistence should be called with a path'); const newDb = await JSONFilePreset(filePath, fallbackData); + const { project, settings, viewSettings, urlPresets, customFields, automation, rundowns } = fallbackData; + await main_db.open(); + await main_db.put('project', project); + await main_db.put('settings', settings); + await main_db.put('viewSettings', viewSettings); + await main_db.put('urlPresets', urlPresets); + await main_db.put('customFields', customFields); + await main_db.put('automation', automation); + + Object.entries(rundowns).forEach(([key, rundown]) => { + rundown_db.put(key, rundown); + }); + // Read the database to initialize it newDb.data = fallbackData; await newDb.write(); @@ -60,6 +86,7 @@ export function getDataProvider() { setAutomation, getRundown, mergeIntoData, + shutdown, }; } @@ -68,13 +95,13 @@ function getData(): Readonly { } async function setProjectData(newData: Partial): ReadonlyPromise { - db.data.project = { ...db.data.project, ...newData }; - await persist(); - return db.data.project; + const newProjectData = { ...getProjectData(), ...newData }; + await main_db.put('project', newProjectData); + return newProjectData; } -function getProjectData(): Readonly { - return db.data.project; +function getProjectData(): ProjectData { + return main_db.getSync('project') as ProjectData; } async function setCustomFields(newData: CustomFields): ReadonlyPromise { @@ -97,8 +124,8 @@ async function mergeRundown( return { rundowns: db.data.rundowns, customFields: db.data.customFields }; } -function getCustomFields(): Readonly { - return db.data.customFields; +function getCustomFields(): CustomFields { + return main_db.getSync('customFields') as CustomFields; } async function setRundown(rundownKey: string, newData: Rundown): ReadonlyPromise { @@ -107,8 +134,8 @@ async function setRundown(rundownKey: string, newData: Rundown): ReadonlyPromise return db.data.rundowns[rundownKey]; } -function getSettings(): Readonly { - return db.data.settings; +function getSettings(): Settings { + return main_db.getSync('settings') as Settings; } async function setSettings(newData: Settings): ReadonlyPromise { @@ -117,8 +144,8 @@ async function setSettings(newData: Settings): ReadonlyPromise { return db.data.settings; } -function getUrlPresets(): Readonly { - return db.data.urlPresets; +function getUrlPresets(): URLPreset[] { + return main_db.getSync('urlPresets') as URLPreset[]; } async function setUrlPresets(newData: URLPreset[]): ReadonlyPromise { @@ -127,8 +154,8 @@ async function setUrlPresets(newData: URLPreset[]): ReadonlyPromise return db.data.urlPresets; } -function getViewSettings(): Readonly { - return db.data.viewSettings; +function getViewSettings(): ViewSettings { + return main_db.getSync('viewSettings'); } async function setViewSettings(newData: ViewSettings): ReadonlyPromise { @@ -137,8 +164,10 @@ async function setViewSettings(newData: ViewSettings): ReadonlyPromise { - return db.data.automation; +function getAutomation(): AutomationSettings { + const automation = main_db.getSync('automation'); + if (!automation) throw new Error('Failed to load automation from db'); + return automation; } async function setAutomation(newData: AutomationSettings): ReadonlyPromise { @@ -147,9 +176,10 @@ async function setAutomation(newData: AutomationSettings): ReadonlyPromise { - const firstRundown = Object.keys(db.data.rundowns)[0]; - return db.data.rundowns[firstRundown]; +function getRundown(): Rundown { + const rundown = rundown_db.getSync('default'); + if (!rundown) throw new Error('Failed to load rundown from db'); + return rundown; } async function mergeIntoData(newData: Partial): ReadonlyPromise { @@ -173,3 +203,8 @@ async function persist() { if (isTest) return; await db.write(); } + +async function shutdown() { + logger.info(LogOrigin.Server, 'Closing DB'); + await main_db.close(); +} diff --git a/apps/server/src/classes/data-provider/levelDb.utils.ts b/apps/server/src/classes/data-provider/levelDb.utils.ts new file mode 100644 index 000000000..0e829f305 --- /dev/null +++ b/apps/server/src/classes/data-provider/levelDb.utils.ts @@ -0,0 +1,24 @@ +export async function batchPutObject(obj: object, db) { + await db.batch( + Object.entries(obj).map(([key, value]) => { + return value === null ? { type: 'del', key } : { type: 'put', key, value }; + }), + ); +} + +// await projectDb.batch([ +// { type: 'put', key: 'title', value: project.title }, +// { type: 'put', key: 'description', value: project.description }, +// { type: 'put', key: 'publicUrl', value: project.publicUrl }, +// { type: 'put', key: 'publicInfo', value: project.publicInfo }, +// { type: 'put', key: 'backstageUrl', value: project.backstageUrl }, +// { type: 'put', key: 'backstageInfo', value: project.backstageInfo }, +// project.backstageInfo +// ? { type: 'put', key: 'projectLogo', value: project.projectLogo } +// : { type: 'del', key: 'projectLogo' }, +// ]); + +// // await levelDb.put('project', project); + +// console.log('level',levelDb.getSync('project')); +// console.log('project',projectDb.getSync('title')); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8ad6219e0..2e5b77f02 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -288,6 +288,9 @@ importers: '@googleapis/sheets': specifier: ^5.0.5 version: 5.0.5 + classic-level: + specifier: ^3.0.0 + version: 3.0.0 cookie: specifier: ^1.0.2 version: 1.0.2 @@ -2272,6 +2275,10 @@ packages: resolution: {integrity: sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==} deprecated: Use your platform's native atob() and btoa() methods instead + abstract-level@3.1.0: + resolution: {integrity: sha512-j2e+TsAxy7Ri+0h7dJqwasymgt0zHBWX4+nMk3XatyuqgHfdstBJ9wsMfbiGwE1O+QovRyPcVAqcViMYdyPaaw==} + engines: {node: '>=18'} + accepts@1.3.8: resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} engines: {node: '>= 0.6'} @@ -2488,6 +2495,9 @@ packages: buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + builder-util-runtime@9.2.4: resolution: {integrity: sha512-upp+biKpN/XZMLim7aguUyW8s0FUpDvOtK6sbanMFDAMBzpHDqdhgVYm6zc9HJ6nWo7u2Lxk60i2M6Jd3aiNrA==} engines: {node: '>=12.0.0'} @@ -2584,6 +2594,10 @@ packages: resolution: {integrity: sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==} engines: {node: '>=8'} + classic-level@3.0.0: + resolution: {integrity: sha512-yGy8j8LjPbN0Bh3+ygmyYvrmskVita92pD/zCoalfcC9XxZj6iDtZTAnz+ot7GG8p9KLTG+MZ84tSA4AhkgVZQ==} + engines: {node: '>=18'} + cli-truncate@2.1.0: resolution: {integrity: sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg==} engines: {node: '>=8'} @@ -3570,6 +3584,10 @@ packages: resolution: {integrity: sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==} engines: {node: '>= 0.4'} + is-buffer@2.0.5: + resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==} + engines: {node: '>=4'} + is-callable@1.2.7: resolution: {integrity: sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==} engines: {node: '>= 0.4'} @@ -3738,6 +3756,14 @@ packages: resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==} engines: {node: '>= 0.6.3'} + level-supports@6.2.0: + resolution: {integrity: sha512-QNxVXP0IRnBmMsJIh+sb2kwNCYcKciQZJEt+L1hPCHrKNELllXhvrlClVHXBYZVT+a7aTSM6StgNXdAldoab3w==} + engines: {node: '>=16'} + + level-transcoder@1.0.1: + resolution: {integrity: sha512-t7bFwFtsQeD8cl8NIoQ2iwxA0CL/9IFw7/9gAjOonH0PWTTiRfY7Hq+Ejbsxh86tXobDQ6IOiddjNYIfOBs06w==} + engines: {node: '>=12'} + levn@0.4.1: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} @@ -3821,6 +3847,10 @@ packages: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} + maybe-combine-errors@1.0.0: + resolution: {integrity: sha512-eefp6IduNPT6fVdwPp+1NgD0PML1NU5P6j1Mj5nz1nidX8/sWY7119WL8vTAHgqfsY74TzW0w1XPgdYEKkGZ5A==} + engines: {node: '>=10'} + media-typer@0.3.0: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} @@ -3917,6 +3947,10 @@ packages: engines: {node: '>=10'} hasBin: true + module-error@1.0.2: + resolution: {integrity: sha512-0yuvsqSCv8LbaOKhnsQ/T5JhyFlCYLPXK3U2sgV10zoKQwzs/MyfuQUOZQ1V/6OCOJsK/TRgNVrPuPDqtdMFtA==} + engines: {node: '>=10'} + ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} @@ -3937,6 +3971,9 @@ packages: engines: {node: ^18 || >=20} hasBin: true + napi-macros@2.2.2: + resolution: {integrity: sha512-hmEVtAGYzVQpCKdbQea4skABsdXW4RUh5t5mJ2zzqowJS2OyXZTU1KhDVFhx+NlWZ4ap9mqR9TcDO3LTTttd+g==} + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -3959,6 +3996,10 @@ packages: encoding: optional: true + node-gyp-build@4.8.4: + resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} + hasBin: true + node-releases@2.0.14: resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==} @@ -7115,6 +7156,15 @@ snapshots: abab@2.0.6: optional: true + abstract-level@3.1.0: + dependencies: + buffer: 6.0.3 + is-buffer: 2.0.5 + level-supports: 6.2.0 + level-transcoder: 1.0.1 + maybe-combine-errors: 1.0.0 + module-error: 1.0.2 + accepts@1.3.8: dependencies: mime-types: 2.1.35 @@ -7396,6 +7446,11 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + builder-util-runtime@9.2.4: dependencies: debug: 4.3.7 @@ -7526,6 +7581,13 @@ snapshots: ci-info@3.9.0: {} + classic-level@3.0.0: + dependencies: + abstract-level: 3.1.0 + module-error: 1.0.2 + napi-macros: 2.2.2 + node-gyp-build: 4.8.4 + cli-truncate@2.1.0: dependencies: slice-ansi: 3.0.0 @@ -8784,6 +8846,8 @@ snapshots: call-bind: 1.0.2 has-tostringtag: 1.0.2 + is-buffer@2.0.5: {} + is-callable@1.2.7: {} is-ci@3.0.1: @@ -8970,6 +9034,13 @@ snapshots: dependencies: readable-stream: 2.3.8 + level-supports@6.2.0: {} + + level-transcoder@1.0.1: + dependencies: + buffer: 6.0.3 + module-error: 1.0.2 + levn@0.4.1: dependencies: prelude-ls: 1.2.1 @@ -9040,6 +9111,8 @@ snapshots: math-intrinsics@1.1.0: {} + maybe-combine-errors@1.0.0: {} + media-typer@0.3.0: {} merge-descriptors@1.0.3: {} @@ -9108,6 +9181,8 @@ snapshots: mkdirp@1.0.4: {} + module-error@1.0.2: {} + ms@2.0.0: {} ms@2.1.3: {} @@ -9126,6 +9201,8 @@ snapshots: nanoid@5.0.7: {} + napi-macros@2.2.2: {} + natural-compare@1.4.0: {} negotiator@0.6.3: {} @@ -9142,6 +9219,8 @@ snapshots: dependencies: whatwg-url: 5.0.0 + node-gyp-build@4.8.4: {} + node-releases@2.0.14: {} normalize-path@3.0.0: {}