Feat resumability (#547)

* add resumability functionality

---------

Co-authored-by: arc-alex <ac@omnivox.dk>
Co-authored-by: Carlos Valente <34649812+cpvalente@users.noreply.github.com>
This commit is contained in:
Alex Christoffer Rasmussen
2023-10-24 08:33:17 +02:00
committed by GitHub
parent b34fe71995
commit a84c0654e2
12 changed files with 453 additions and 33 deletions
+1 -1
View File
@@ -20,7 +20,7 @@
},
"scripts": {
"postinstall": "",
"dev:electron": "NODE_ENV=development electron .",
"dev:electron": "cross-env NODE_ENV=development electron .",
"dist-win": "electron-builder --publish=never --x64 --win",
"dist-mac": "electron-builder --publish=never --mac",
"dist-linux": "electron-builder --publish=never --x64 --linux",
+3 -2
View File
@@ -19,6 +19,7 @@
"ontime-utils": "workspace:*",
"passport": "^0.6.0",
"passport-local": "~1.0.0",
"steno": "^3.1.0",
"ws": "^8.13.0"
},
"devDependencies": {
@@ -52,8 +53,8 @@
"build:docker": "pnpm prebuild && esbuild src/index.ts --log-level=error --platform=node --format=cjs --minify --bundle --outfile=dist/docker.cjs",
"build:debug": "pnpm prebuild && esbuild src/app.ts --platform=node --format=cjs --bundle --outfile=dist/index.cjs",
"lint": "eslint .",
"test": "vitest",
"test:pipeline": "vitest run",
"test": "cross-env IS_TEST=true vitest",
"test:pipeline": "cross-env IS_TEST=true vitest run",
"typecheck": "tsc --noEmit",
"cleanup": "rm -rf .turbo && rm -rf node_modules && rm -rf dist"
}
+25 -4
View File
@@ -1,3 +1,5 @@
import { LogOrigin, OSCSettings } from 'ontime-types';
import 'dotenv/config';
import express from 'express';
import expressStaticGzip from 'express-static-gzip';
@@ -6,10 +8,8 @@ import cors from 'cors';
// import utils
import { join, resolve } from 'path';
import { currentDirectory, environment, externalsStartDirectory, isProduction, resolvedPath } from './setup.js';
import { ONTIME_VERSION } from './ONTIME_VERSION.js';
import { LogOrigin, OSCSettings } from 'ontime-types';
// Import Routes
import { router as rundownRouter } from './routes/rundownRouter.js';
@@ -31,6 +31,8 @@ import { logger } from './classes/Logger.js';
import { oscIntegration } from './services/integration-service/OscIntegration.js';
import { populateStyles } from './modules/loadStyles.js';
import { eventStore, getInitialPayload } from './stores/EventStore.js';
import { PlaybackService } from './services/PlaybackService.js';
import { RestorePoint, restoreService } from './services/RestoreService.js';
console.log(`Starting Ontime version ${ONTIME_VERSION}`);
@@ -137,10 +139,21 @@ export const startServer = async () => {
expressServer = http.createServer(app);
socket.init(expressServer);
eventLoader.init();
// load restore point if it exists
const maybeRestorePoint = restoreService.load();
if (maybeRestorePoint) {
logger.info(LogOrigin.Server, 'Found resumable state');
PlaybackService.resume(maybeRestorePoint);
}
eventTimer.setRestoreCallback(async (newState: RestorePoint) => restoreService.save(newState));
// provide initial payload to event store
eventLoader.init();
eventStore.init(getInitialPayload());
const initialPayload = getInitialPayload();
eventStore.init(initialPayload);
expressServer.listen(serverPort, '0.0.0.0');
@@ -202,6 +215,14 @@ export const startIntegrations = async (config?: { osc: OSCSettings }) => {
export const shutdown = async (exitCode = 0) => {
console.log(`Ontime shutting down with code ${exitCode}`);
// clear the restore file if it was a normal exit
// 0 means it was a SIGNAL
// 1 means crash -> keep the file
// 99 means it was the UI
if (exitCode === 0 || exitCode === 99) {
await restoreService.clear();
}
expressServer?.close();
oscServer?.shutdown();
eventTimer.shutdown();
@@ -23,6 +23,18 @@ export class EventLoader {
// eslint-disable-next-line @typescript-eslint/no-this-alias -- this logic is used to ensure singleton
instance = this;
this.eventNow = null;
this.publicEventNow = null;
this.eventNext = null;
this.publicEventNext = null;
this.loaded = {
selectedEventIndex: null,
selectedEventId: null,
selectedPublicEventId: null,
nextEventId: null,
nextPublicEventId: null,
numEvents: 0,
};
}
// we need to delay init until the store is ready
@@ -71,7 +83,7 @@ export class EventLoader {
* @param {string} eventId
* @return {object | undefined}
*/
static getEventWithId(eventId) {
static getEventWithId(eventId): OntimeEvent | undefined {
const timedEvents = EventLoader.getTimedEvents();
return timedEvents.find((event) => event.id === eventId);
}
@@ -223,7 +235,7 @@ export class EventLoader {
* loads an event given its id
* @param {object} event
*/
loadEvent(event) {
loadEvent(event?: OntimeEvent) {
if (typeof event === 'undefined') {
return null;
}
+1 -1
View File
@@ -3,10 +3,10 @@ export const config = {
testdb: 'test-db',
directory: 'db',
filename: 'db.json',
tablename: 'events',
},
styles: {
directory: 'styles',
filename: 'override.css',
},
restoreFile: 'ontime.restore',
};
+29 -1
View File
@@ -1,4 +1,4 @@
import { LogOrigin, OntimeEvent } from 'ontime-types';
import { LogOrigin, OntimeEvent, Playback } from 'ontime-types';
import { validatePlayback } from 'ontime-utils';
import { eventLoader, EventLoader } from '../classes/event-loader/EventLoader.js';
@@ -6,6 +6,7 @@ import { eventStore } from '../stores/EventStore.js';
import { eventTimer } from './TimerService.js';
import { clock } from './Clock.js';
import { logger } from '../classes/Logger.js';
import { RestorePoint } from './RestoreService.js';
/**
* Service manages playback status of app
@@ -241,6 +242,33 @@ export class PlaybackService {
}
}
/**
* @description resume playback state given a restore point
* @param restorePoint
*/
static resume(restorePoint: RestorePoint) {
const willResume = () => logger.info(LogOrigin.Server, 'Resuming playback');
if (restorePoint.playback === Playback.Roll) {
willResume();
PlaybackService.roll();
}
if (restorePoint.selectedEventId) {
const event = EventLoader.getEventWithId(restorePoint.selectedEventId);
// the db would have to change for the event not to exist
// we do not kow the reason for the crash, so we check anyway
if (!event) {
return;
}
eventLoader.loadEvent(event);
eventTimer.resume(event, restorePoint);
eventStore.broadcast();
return;
}
}
/**
* Adds delay to current event
* @param {number} delayTime time in minutes
+160
View File
@@ -0,0 +1,160 @@
import { Playback } from 'ontime-types';
import { readFileSync } from 'fs';
import { Writer } from 'steno';
import { resolveRestoreFile } from '../setup.js';
export type RestorePoint = {
playback: Playback;
selectedEventId: string | null;
startedAt: number | null;
addedTime: number | null;
pausedAt: number | null;
};
/**
* Utility validates a RestorePoint
* @param obj
* @return boolean
*/
export function isRestorePoint(obj: unknown): obj is RestorePoint {
if (!obj) {
return false;
}
const restorePoint = obj as RestorePoint;
if (typeof restorePoint.playback !== 'string' || !Object.values(Playback).includes(restorePoint.playback)) {
return false;
}
if (typeof restorePoint.selectedEventId !== 'string' && restorePoint.selectedEventId !== null) {
return false;
}
if (typeof restorePoint.startedAt !== 'number' && restorePoint.startedAt !== null) {
return false;
}
if (typeof restorePoint.addedTime !== 'number' && restorePoint.addedTime !== null) {
return false;
}
if (typeof restorePoint.pausedAt !== 'number' && restorePoint.pausedAt !== null) {
return false;
}
return true;
}
/**
* Utility interface to allow dependency injection during test
*/
/**
* Service manages saving of application state
* that can then be restored when reopening
*/
export class RestoreService {
private readonly filePath: string | null;
private lastStore: string | null;
private file: Writer | null;
private failedCreateAttempts: number;
constructor(filePath: string) {
this.filePath = filePath;
this.lastStore = null;
this.file = null;
this.failedCreateAttempts = 0;
}
/**
* Utility, creates a restore file
*/
create() {
this.file = new Writer(this.filePath);
}
/**
* Utility, reads from file
* @private
*/
private read() {
return readFileSync(this.filePath, 'utf-8');
}
/**
* Utility writes payload to file
* @throws
* @param stringifiedState
*/
private async write(stringifiedState: string) {
// Create a file if it doesnt exist
if (!this.file) {
this.create();
}
// steno is async, and it uses a queue to avoid unnecessary re-writes
await this.file.write(stringifiedState);
}
/**
* Saves runtime data to restore file
* @param newState RestorePoint
*/
async save(newState: RestorePoint) {
// after three failed attempts, mark the service as unavailable
if (this.failedCreateAttempts > 3) {
return;
}
const stringifiedStore = JSON.stringify(newState);
if (stringifiedStore !== this.lastStore) {
try {
await this.write(stringifiedStore);
this.lastStore = stringifiedStore;
this.failedCreateAttempts = 0;
} catch (_err) {
this.failedCreateAttempts += 1;
}
}
}
/**
* Attempts reading a restore point from a given file path
* Returns null if none found, restore point otherwise
*/
load(): RestorePoint | null {
try {
const data = this.read();
const maybeRestorePoint = JSON.parse(data);
if (!isRestorePoint(maybeRestorePoint)) {
return null;
}
return maybeRestorePoint;
} catch (_error) {
// no need to notify the user
return null;
}
}
/**
* Clears the restore file
*/
async clear() {
if (this.file && this.failedCreateAttempts <= 3) {
try {
await this.file.write('');
} catch (_error) {
// nothing to do
}
}
this.file = undefined;
}
}
export const restoreService = new RestoreService(resolveRestoreFile);
+75 -13
View File
@@ -1,4 +1,4 @@
import { EndAction, OntimeEvent, Playback, TimerLifeCycle, TimerState, TimerType } from 'ontime-types';
import { EndAction, LogOrigin, OntimeEvent, Playback, TimerLifeCycle, TimerState, TimerType } from 'ontime-types';
import { calculateDuration, dayInMs } from 'ontime-utils';
import { eventStore } from '../stores/EventStore.js';
@@ -8,6 +8,7 @@ import { integrationService } from './integration-service/IntegrationService.js'
import { getCurrent, getExpectedFinish } from './timerUtils.js';
import { clock } from './Clock.js';
import { logger } from '../classes/Logger.js';
import type { RestorePoint } from './RestoreService.js';
type initialLoadingData = {
startedAt?: number | null;
@@ -15,6 +16,8 @@ type initialLoadingData = {
current?: number | null;
};
type RestoreCallback = (newState: RestorePoint) => Promise<void>;
export class TimerService {
private readonly _interval: NodeJS.Timer;
private _updateInterval: number;
@@ -31,6 +34,7 @@ export class TimerService {
private pausedAt: number | null;
private secondaryTarget: number | null;
private saveRestorePoint: RestoreCallback;
/**
* @constructor
* @param {object} [timerConfig]
@@ -43,6 +47,14 @@ export class TimerService {
this._updateInterval = timerConfig?.updateInterval ?? 1000;
}
/**
* Provides callback to save restore point
* @param cb
*/
setRestoreCallback(cb: RestoreCallback) {
this.saveRestorePoint = cb;
}
/**
* Clears internal state
* @private
@@ -74,6 +86,44 @@ export class TimerService {
this._lastUpdate = null;
}
/**
* Resumes a given playback state, same as load
* @param {RestorePoint} restorePoint
* @param {OntimeEvent} timer
*/
resume(timer: OntimeEvent, restorePoint: RestorePoint) {
this._clear();
// this is pretty much the same as load, with a few exceptions
this.loadedTimerId = timer.id;
this.loadedTimerStart = timer.timeStart;
this.loadedTimerEnd = timer.timeEnd;
this.timer.duration = calculateDuration(timer.timeStart, timer.timeEnd);
this.playback = restorePoint.playback;
this.timer.timerType = timer.timerType;
this.timer.endAction = timer.endAction;
this.timer.startedAt = restorePoint.startedAt;
this.timer.addedTime = restorePoint.addedTime;
this.pausedTime = 0;
this.pausedAt = restorePoint.pausedAt;
this.timer.current = this.timer.duration;
if (this.timer.timerType === TimerType.TimeToEnd) {
const now = clock.timeNow();
this.timer.current = getCurrent(now, this.timer.duration, 0, 0, now, timer.timeEnd, this.timer.timerType);
}
this._onResume();
}
_onResume() {
eventStore.batchSet({
playback: this.playback,
timer: this.timer,
});
}
/**
* Reloads information for currently running timer
* @param timer
@@ -121,17 +171,10 @@ export class TimerService {
/**
* Loads given timer to object
* @param {object} timer
* @param initialData
* @param {number} timer.id
* @param {number} timer.timeStart
* @param {number} timer.timeEnd
* @param {number} timer.duration
* @param {string} timer.timerBehaviour
* @param {string} timer.timerType
* @param {boolean} timer.skip
* @param {OntimeEvent} timer
* @param {initialLoadingData} initialData
*/
load(timer, initialData?: initialLoadingData) {
load(timer: OntimeEvent, initialData?: initialLoadingData) {
if (timer.skip) {
throw new Error('Refuse load of skipped event');
}
@@ -155,7 +198,7 @@ export class TimerService {
this.timer.current = getCurrent(now, this.timer.duration, 0, 0, now, timer.timeEnd, this.timer.timerType);
}
if (typeof initialData !== 'undefined') {
if (initialData) {
this.timer = { ...this.timer, ...initialData };
}
@@ -172,12 +215,13 @@ export class TimerService {
timer: this.timer,
});
integrationService.dispatch(TimerLifeCycle.onLoad);
this._saveState();
}
start() {
if (!this.loadedTimerId) {
if (this.playback === Playback.Roll) {
logger.error('PLAYBACK', 'Cannot start while waiting for event');
logger.error(LogOrigin.Playback, 'Cannot start while waiting for event');
}
return;
}
@@ -222,6 +266,7 @@ export class TimerService {
timer: this.timer,
});
integrationService.dispatch(TimerLifeCycle.onStart);
this._saveState();
}
pause() {
@@ -237,6 +282,7 @@ export class TimerService {
timer: this.timer,
});
integrationService.dispatch(TimerLifeCycle.onPause);
this._saveState();
}
stop() {
@@ -254,6 +300,7 @@ export class TimerService {
timer: this.timer,
});
integrationService.dispatch(TimerLifeCycle.onStop);
this._saveState();
}
/**
@@ -280,6 +327,7 @@ export class TimerService {
// force an update
this.update(true);
this._saveState();
}
private updateRoll() {
@@ -394,6 +442,7 @@ export class TimerService {
PlaybackService.startNext();
}
}
this._saveState();
}
/**
@@ -435,6 +484,19 @@ export class TimerService {
_onRoll() {
eventStore.set('playback', this.playback);
this._saveState();
}
async _saveState() {
if (this.saveRestorePoint) {
await this.saveRestorePoint({
playback: this.playback,
selectedEventId: this.loadedTimerId,
startedAt: this.timer.startedAt,
addedTime: this.timer.addedTime,
pausedAt: this.pausedAt,
});
}
}
shutdown() {
@@ -0,0 +1,128 @@
import { describe, expect, it, vi } from 'vitest';
import { Playback } from 'ontime-types';
import { isRestorePoint, RestorePoint, RestoreService } from '../RestoreService.js';
describe('isRestorePoint()', () => {
it('validates a well defined object', () => {
let restorePoint = {
playback: 'play',
selectedEventId: '123',
startedAt: 1,
addedTime: 2,
pausedAt: 3,
};
expect(isRestorePoint(restorePoint)).toBe(true);
restorePoint = {
playback: 'roll',
selectedEventId: '123',
startedAt: null,
addedTime: null,
pausedAt: null,
};
expect(isRestorePoint(restorePoint)).toBe(true);
});
describe('rejects a badly formatted file', () => {
it('with invalid playback value', () => {
const restorePoint = {
playback: 'unknown',
selectedEventId: '123',
startedAt: null,
addedTime: null,
pausedAt: null,
};
expect(isRestorePoint(restorePoint)).toBe(false);
});
it('with missing playback value', () => {
const restorePoint = {
selectedEventId: '123',
startedAt: null,
addedTime: null,
pausedAt: null,
};
expect(isRestorePoint(restorePoint)).toBe(false);
});
it('with incorrect value', () => {
const restorePoint = {
playback: 'roll',
selectedEventId: '123',
startedAt: 'testing',
addedTime: null,
pausedAt: null,
};
expect(isRestorePoint(restorePoint)).toBe(false);
});
});
});
describe('RestoreService()', () => {
describe('load()', () => {
it('loads working file with times', () => {
const expected = {
playback: Playback.Play,
selectedEventId: 'da5b4',
startedAt: 1234,
addedTime: 5678,
pausedAt: 9087,
};
const restoreService = new RestoreService('/path/to/restore/file');
vi.spyOn<any, any>(restoreService, 'read').mockImplementation(() => JSON.stringify(expected));
const testLoad = restoreService.load();
expect(testLoad).toStrictEqual(expected);
});
it('loads working file without times', () => {
const expected = {
playback: Playback.Stop,
selectedEventId: null,
startedAt: null,
addedTime: null,
pausedAt: null,
};
const restoreService = new RestoreService('/path/to/restore/file');
vi.spyOn<any, any>(restoreService, 'read').mockImplementation(() => JSON.stringify(expected));
const testLoad = restoreService.load();
expect(testLoad).toStrictEqual(expected);
});
it('does not load wrong play state', () => {
const expected = {
playback: 'does-not-exist',
selectedEventId: 'da5b4',
startedAt: 1234,
addedTime: 1234,
pausedAt: 1234,
};
const restoreService = new RestoreService('/path/to/restore/file');
vi.spyOn<any, any>(restoreService, 'read').mockImplementation(() => JSON.stringify(expected));
const testLoad = restoreService.load();
expect(testLoad).toBe(null);
});
});
describe('save()', () => {
it('saves data to file', async () => {
const testData: RestorePoint = {
playback: Playback.Play,
selectedEventId: '1234',
startedAt: 1234,
addedTime: 1234,
pausedAt: 1234,
};
const restoreService = new RestoreService('/path/to/restore/file');
const writeSpy = vi.spyOn<any, any>(restoreService, 'write').mockImplementation(() => undefined);
await restoreService.save(testData);
expect(writeSpy).toHaveBeenCalledWith(JSON.stringify(testData));
});
});
});
+3
View File
@@ -85,3 +85,6 @@ export const resolveStylesDirectory = join(externalsStartDirectory, config.style
export const resolveStylesPath = join(resolveStylesDirectory, config.styles.filename);
export const pathToStartStyles = join(currentDirectory, '/external/styles/', config.styles.filename);
// path to restore file
export const resolveRestoreFile = join(getAppDataPath(), config.restoreFile);
+7 -4
View File
@@ -25,7 +25,8 @@
"user9": "",
"type": "event",
"revision": 0,
"id": "aa42f"
"id": "aa42f",
"cue": "1"
},
{
"duration": 600000,
@@ -58,7 +59,8 @@
"user9": "",
"type": "event",
"revision": 0,
"id": "d71bc"
"id": "d71bc",
"cue": "2"
},
{
"title": "Lunch",
@@ -74,7 +76,7 @@
"timerType": "count-down",
"timeStart": 39600000,
"timeEnd": 720000,
"duration": 0,
"duration": 47520000,
"isPublic": true,
"skip": false,
"colour": "",
@@ -90,7 +92,8 @@
"user9": "",
"type": "event",
"revision": 0,
"id": "da5b4"
"id": "da5b4",
"cue": "3"
}
],
"project": {
+7 -5
View File
@@ -196,6 +196,7 @@ importers:
passport-local: ~1.0.0
prettier: ^2.8.3
shx: ^0.3.4
steno: ^3.1.0
ts-node: ^10.9.1
typescript: ^4.9.4
vitest: ^0.30.1
@@ -215,6 +216,7 @@ importers:
ontime-utils: link:../../packages/utils
passport: 0.6.0
passport-local: 1.0.0
steno: 3.1.0
ws: 8.13.0
devDependencies:
'@types/express': 4.17.17
@@ -6012,7 +6014,7 @@ packages:
resolution: {integrity: sha512-7EWKmIMhNKA8TXFhL8t0p6N2LC53l3ZqsWQGSksGhhjrcms9rbKlyrAh2PzSGK5v0KPJ2W5VItBnC3NDRzOnzQ==}
engines: {node: '>=14.16'}
dependencies:
steno: 3.0.0
steno: 3.1.0
dev: false
/lowercase-keys/2.0.0:
@@ -7327,9 +7329,9 @@ packages:
resolution: {integrity: sha512-uUZI65yrV2Qva5gqE0+A7uVAvO40iPo6jGhs7s8keRfHCmtg+uB2X6EiLGCI9IgL1J17xGhvoOqSz79lzICPTA==}
dev: true
/steno/3.0.0:
resolution: {integrity: sha512-uZtn7Ht9yXLiYgOsmo8btj4+f7VxyYheMt8g6F1ANjyqByQXEE2Gygjgenp3otHH1TlHsS4JAaRGv5wJ1wvMNw==}
engines: {node: '>=14.16'}
/steno/3.1.0:
resolution: {integrity: sha512-U9mIkOthSBoLxa+4QAXv0aDDHeLn6merFMkjSblSz+WgezKQ0EkS1znRY6hNBZz3kGDm/0ZaP+E+/1X1ho37IQ==}
engines: {node: '>=16'}
dev: false
/stop-iteration-iterator/1.0.0:
@@ -8500,4 +8502,4 @@ packages:
dependencies:
react: 18.2.0
use-sync-external-store: 1.2.0_react@18.2.0
dev: false
dev: false